From 1652787ec9114074e0e97aec93340b9af6c34dfa Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:56:21 +0000 Subject: [PATCH 01/64] Propose test harness resource cleanup with worker evidence --- .../filesystem-inspection.txt | 32 + .../gc-renderers.json | 578 ++++ .../measurements.json | 2585 +++++++++++++++++ .../run3-kernel-oom.txt | 8 + .../run4-kernel-oom.txt | 11 + .../chromium-insufficient-resources.md | 107 + .../tv-testshard-x64-poc-03-sizes.json | 35 + .../tv-testshard-x64-poc-03.json | 420 +++ .../tv-testshard-x64-poc-131-sizes.json | 39 + .../tv-testshard-x64-poc-131.json | 502 ++++ .../test-harness-resource-cleanup-proposal.md | 154 + 11 files changed, 4471 insertions(+) create mode 100644 docs/working/chromium-insufficient-resources-evidence/filesystem-inspection.txt create mode 100644 docs/working/chromium-insufficient-resources-evidence/gc-renderers.json create mode 100644 docs/working/chromium-insufficient-resources-evidence/measurements.json create mode 100644 docs/working/chromium-insufficient-resources-evidence/run3-kernel-oom.txt create mode 100644 docs/working/chromium-insufficient-resources-evidence/run4-kernel-oom.txt create mode 100644 docs/working/chromium-insufficient-resources.md create mode 100644 docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03-sizes.json create mode 100644 docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03.json create mode 100644 docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131-sizes.json create mode 100644 docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131.json create mode 100644 docs/working/test-harness-resource-cleanup-proposal.md diff --git a/docs/working/chromium-insufficient-resources-evidence/filesystem-inspection.txt b/docs/working/chromium-insufficient-resources-evidence/filesystem-inspection.txt new file mode 100644 index 00000000..bb656303 --- /dev/null +++ b/docs/working/chromium-insufficient-resources-evidence/filesystem-inspection.txt @@ -0,0 +1,32 @@ +Read-only inspection of tv-testshard-x64-poc-131, 2026-10-01. +This is later state, not a snapshot at the original failed run's instant. + +find /tmp -maxdepth 1 -type d -name 'television-source-cli-e2e-*' | wc -l +267 + +du -h -d 1 /tmp /workspace/television /cache /home/playwright/.cache /home/playwright/.nvm +Selected output: +13M /tmp/node-compile-cache +17M /tmp/playwright-transform-cache-0 +1.2G /tmp +167M /cache/npm +241M /cache/electron +407M /cache +243M /workspace/television/.git +1.1G /workspace/television/node_modules +1.4G /workspace/television +234M /home/playwright/.nvm +915M /home/playwright/.cache/ms-playwright +1.1G /home/playwright/.cache + +Source CLI runtime directories in the largest listed entries were approximately +4.1M each. getSourceCLIEntry() has no cleanup of their shared build/runtime tree. + +df -h /tmp /dev/shm +Filesystem Size Used Avail Use% Mounted on +overlay 6.3G 4.3G 2.0G 68% / +tmpfs 3.9G 512K 3.9G 1% /dev/shm + +/proc/mounts, relevant rows: +overlay / overlay rw,relatime,lowerdir=/mnt/erofs,upperdir=/mnt/tmp/upper,workdir=/mnt/tmp/work,uuid=on 0 0 +tmpfs /dev/shm tmpfs rw,nosuid,noexec,relatime 0 0 diff --git a/docs/working/chromium-insufficient-resources-evidence/gc-renderers.json b/docs/working/chromium-insufficient-resources-evidence/gc-renderers.json new file mode 100644 index 00000000..5eb571b2 --- /dev/null +++ b/docs/working/chromium-insufficient-resources-evidence/gc-renderers.json @@ -0,0 +1,578 @@ +[ + { + "kind": "ROSE_GC_BEFORE", + "data": { + "at": "2026-10-01T22:21:26.951Z", + "processes": [ + { + "pid": 11694, + "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1519017284 kB", + "VmRSS:\t 94780 kB", + "Threads:\t15" + ], + "fdCount": 259, + "limits": [ + "Max processes 31819 31819 processes ", + "Max open files 4096 4096 files ", + "Max address space unlimited unlimited bytes " + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "shared-memory": 229, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4 + }, + "sharedMemorySizes": { + "131000": 8, + "2097152": 220, + "4198400": 1 + }, + "sharedMemoryAllocated": 466620416 + }, + { + "pid": 11730, + "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1519015940 kB", + "VmRSS:\t 86480 kB", + "Threads:\t15" + ], + "fdCount": 259, + "limits": [ + "Max processes 31819 31819 processes ", + "Max open files 4096 4096 files ", + "Max address space unlimited unlimited bytes " + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "shared-memory": 229, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4 + }, + "sharedMemorySizes": { + "131000": 8, + "2097152": 220, + "4198400": 1 + }, + "sharedMemoryAllocated": 466620416 + } + ], + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2730892 kB", + "MemAvailable: 2724816 kB", + "SwapFree: 0 kB", + "Shmem: 3822512 kB" + ], + "fileNr": "832\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 2762153984 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + } + } + }, + { + "kind": "ROSE_GC_AFTER", + "data": { + "at": "2026-10-01T22:21:27.072Z", + "processes": [ + { + "pid": 11694, + "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1518566212 kB", + "VmRSS:\t 55728 kB", + "Threads:\t15" + ], + "fdCount": 35, + "limits": [ + "Max processes 31819 31819 processes ", + "Max open files 4096 4096 files ", + "Max address space unlimited unlimited bytes " + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4, + "shared-memory": 5 + }, + "sharedMemorySizes": { + "131000": 4, + "4198400": 1 + }, + "sharedMemoryAllocated": 4722688 + }, + { + "pid": 11730, + "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1518564868 kB", + "VmRSS:\t 52288 kB", + "Threads:\t15" + ], + "fdCount": 35, + "limits": [ + "Max processes 31819 31819 processes ", + "Max open files 4096 4096 files ", + "Max address space unlimited unlimited bytes " + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4, + "shared-memory": 5 + }, + "sharedMemorySizes": { + "131000": 4, + "4198400": 1 + }, + "sharedMemoryAllocated": 4722688 + } + ], + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3699996 kB", + "MemAvailable: 3692256 kB", + "SwapFree: 0 kB", + "Shmem: 2920608 kB" + ], + "fileNr": "416\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 3685933056 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + } + } + }, + { + "kind": "ROSE_GC_BEFORE", + "data": { + "at": "2026-10-01T22:21:28.387Z", + "processes": [ + { + "pid": 11694, + "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1518575396 kB", + "VmRSS:\t 86100 kB", + "Threads:\t15" + ], + "fdCount": 43, + "limits": [ + "Max processes 31819 31819 processes ", + "Max open files 4096 4096 files ", + "Max address space unlimited unlimited bytes " + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4, + "shared-memory": 13 + }, + "sharedMemorySizes": { + "131000": 8, + "2097152": 4, + "4198400": 1 + }, + "sharedMemoryAllocated": 13635584 + }, + { + "pid": 11730, + "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1518573572 kB", + "VmRSS:\t 80040 kB", + "Threads:\t15" + ], + "fdCount": 43, + "limits": [ + "Max processes 31819 31819 processes ", + "Max open files 4096 4096 files ", + "Max address space unlimited unlimited bytes " + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4, + "shared-memory": 13 + }, + "sharedMemorySizes": { + "131000": 8, + "2097152": 4, + "4198400": 1 + }, + "sharedMemoryAllocated": 13635584 + } + ], + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3506724 kB", + "MemAvailable: 3498712 kB", + "SwapFree: 0 kB", + "Shmem: 2944532 kB" + ], + "fileNr": "448\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 3661377536 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + } + } + }, + { + "kind": "ROSE_GC_AFTER", + "data": { + "at": "2026-10-01T22:21:28.498Z", + "processes": [ + { + "pid": 11694, + "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1518566692 kB", + "VmRSS:\t 58788 kB", + "Threads:\t15" + ], + "fdCount": 35, + "limits": [ + "Max processes 31819 31819 processes ", + "Max open files 4096 4096 files ", + "Max address space unlimited unlimited bytes " + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4, + "shared-memory": 5 + }, + "sharedMemorySizes": { + "131000": 4, + "4198400": 1 + }, + "sharedMemoryAllocated": 4722688 + }, + { + "pid": 11730, + "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1518564868 kB", + "VmRSS:\t 54528 kB", + "Threads:\t15" + ], + "fdCount": 35, + "limits": [ + "Max processes 31819 31819 processes ", + "Max open files 4096 4096 files ", + "Max address space unlimited unlimited bytes " + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4, + "shared-memory": 5 + }, + "sharedMemorySizes": { + "131000": 4, + "4198400": 1 + }, + "sharedMemoryAllocated": 4722688 + } + ], + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3542992 kB", + "MemAvailable: 3534980 kB", + "SwapFree: 0 kB", + "Shmem: 2927288 kB" + ], + "fileNr": "416\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 3679182848 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + } + } + }, + { + "kind": "ROSE_GC_BEFORE", + "data": { + "at": "2026-10-01T22:21:29.888Z", + "processes": [ + { + "pid": 11694, + "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1518584072 kB", + "VmRSS:\t 87392 kB", + "Threads:\t16" + ], + "fdCount": 43, + "limits": [ + "Max processes 31819 31819 processes ", + "Max open files 4096 4096 files ", + "Max address space unlimited unlimited bytes " + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4, + "shared-memory": 13 + }, + "sharedMemorySizes": { + "131000": 8, + "2097152": 4, + "4198400": 1 + }, + "sharedMemoryAllocated": 13635584 + }, + { + "pid": 11730, + "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1518573572 kB", + "VmRSS:\t 81144 kB", + "Threads:\t15" + ], + "fdCount": 43, + "limits": [ + "Max processes 31819 31819 processes ", + "Max open files 4096 4096 files ", + "Max address space unlimited unlimited bytes " + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4, + "shared-memory": 13 + }, + "sharedMemorySizes": { + "131000": 8, + "2097152": 4, + "4198400": 1 + }, + "sharedMemoryAllocated": 13635584 + } + ], + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3332724 kB", + "MemAvailable: 3324684 kB", + "SwapFree: 0 kB", + "Shmem: 2951136 kB" + ], + "fileNr": "448\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 3654643712 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + } + } + }, + { + "kind": "ROSE_GC_AFTER", + "data": { + "at": "2026-10-01T22:21:29.953Z", + "processes": [ + { + "pid": 11694, + "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1518575368 kB", + "VmRSS:\t 59208 kB", + "Threads:\t16" + ], + "fdCount": 35, + "limits": [ + "Max processes 31819 31819 processes ", + "Max open files 4096 4096 files ", + "Max address space unlimited unlimited bytes " + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4, + "shared-memory": 5 + }, + "sharedMemorySizes": { + "131000": 4, + "4198400": 1 + }, + "sharedMemoryAllocated": 4722688 + }, + { + "pid": 11730, + "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1518564868 kB", + "VmRSS:\t 55196 kB", + "Threads:\t15" + ], + "fdCount": 35, + "limits": [ + "Max processes 31819 31819 processes ", + "Max open files 4096 4096 files ", + "Max address space unlimited unlimited bytes " + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4, + "shared-memory": 5 + }, + "sharedMemorySizes": { + "131000": 4, + "4198400": 1 + }, + "sharedMemoryAllocated": 4722688 + } + ], + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3341408 kB", + "MemAvailable: 3333368 kB", + "SwapFree: 0 kB", + "Shmem: 2933852 kB" + ], + "fileNr": "448\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 3672453120 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + } + } + } +] diff --git a/docs/working/chromium-insufficient-resources-evidence/measurements.json b/docs/working/chromium-insufficient-resources-evidence/measurements.json new file mode 100644 index 00000000..632844d3 --- /dev/null +++ b/docs/working/chromium-insufficient-resources-evidence/measurements.json @@ -0,0 +1,2585 @@ +[ + { + "experiment": 1, + "runDirectory": "/home/user/workspace/wt/television/investigate-chromium-insufficient-resources/.test-runs/2026-10-01T22-06-53-142Z-p2875221-r19ba7810391cbf6c", + "commit": "0ba2dcdaed7c9a2d7692be012255d4b5ceb7d5af", + "sandbox": "tv-testshard-x64-poc-143", + "result": { + "id": "2026-10-01T22-06-53-142Z-p2875221-r19ba7810391cbf6c", + "commandProvider": "blaxel", + "timingProvider": "blaxel-playwright-x64-4vcpu", + "status": "passed", + "infraStatus": "completed", + "testStatus": "passed", + "startedAt": "2026-10-01T22:06:54.717Z", + "completedAt": "2026-10-01T22:07:40.426Z", + "durationMs": 45709 + }, + "attempts": [ + { + "title": "executable theme reset reloads connected Chromium documents under confirmed state", + "retry": 0, + "status": "passed", + "durationMs": 5959, + "startTime": "2026-10-01T22:07:31.091Z", + "error": null, + "measurements": [ + { + "kind": "start", + "data": { + "at": "2026-10-01T22:07:31.311Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3734932 kB", + "MemAvailable: 3745952 kB", + "SwapFree: 0 kB", + "Shmem: 3079096 kB" + ], + "fileNr": "224\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 7478, + "kind": "network", + "fdCount": 20, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7458", + "FDSize:\t128", + "VmSize:\t50617288 kB", + "VmRSS:\t 8840 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 4, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1 + } + } + ] + } + }, + { + "kind": "peak", + "data": { + "maximumFD": 707, + "maximumPending": 261, + "failures": 0, + "sample": { + "at": "2026-10-01T22:07:37.108Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 528056 kB", + "MemAvailable: 549428 kB", + "SwapFree: 0 kB", + "Shmem: 5840544 kB" + ], + "fileNr": "1696\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 7478, + "kind": "network", + "fdCount": 42, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7458", + "FDSize:\t128", + "VmSize:\t50618576 kB", + "VmRSS:\t 20044 kB", + "Threads:\t8" + ] + }, + { + "pid": 7502, + "kind": "renderer", + "fdCount": 707, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7461", + "FDSize:\t1024", + "VmSize:\t1519911712 kB", + "VmRSS:\t 140560 kB", + "Threads:\t14" + ] + }, + { + "pid": 7540, + "kind": "renderer", + "fdCount": 707, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7461", + "FDSize:\t1024", + "VmSize:\t1519909888 kB", + "VmRSS:\t 127952 kB", + "Threads:\t14" + ] + } + ] + } + } + }, + { + "kind": "end", + "data": { + "at": "2026-10-01T22:07:37.233Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 801948 kB", + "MemAvailable: 823440 kB", + "SwapFree: 0 kB", + "Shmem: 5635056 kB" + ], + "fileNr": "992\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 7478, + "kind": "network", + "fdCount": 28, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7458", + "FDSize:\t128", + "VmSize:\t50617420 kB", + "VmRSS:\t 19576 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 11, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 1 + } + }, + { + "pid": 7502, + "kind": "renderer", + "fdCount": 707, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7461", + "FDSize:\t1024", + "VmSize:\t1519912048 kB", + "VmRSS:\t 141920 kB", + "Threads:\t14" + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "shared-memory": 677, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4 + } + } + ] + } + } + ], + "failedRequestCount": 0, + "firstFailedRequests": [] + } + ] + }, + { + "experiment": 2, + "runDirectory": "/home/user/workspace/wt/television/investigate-chromium-insufficient-resources/.test-runs/2026-10-01T22-08-43-298Z-p2875805-r39e7ed872b6bcb21", + "commit": "0ba2dcdaed7c9a2d7692be012255d4b5ceb7d5af", + "sandbox": "tv-testshard-x64-poc-39", + "result": { + "id": "2026-10-01T22-08-43-298Z-p2875805-r39e7ed872b6bcb21", + "commandProvider": "blaxel", + "timingProvider": "blaxel-playwright-x64-4vcpu", + "status": "passed", + "infraStatus": "completed", + "testStatus": "passed", + "startedAt": "2026-10-01T22:08:44.865Z", + "completedAt": "2026-10-01T22:09:29.186Z", + "durationMs": 44321 + }, + "attempts": [ + { + "title": "executable theme reset reloads connected Chromium documents under confirmed state", + "retry": 0, + "status": "passed", + "durationMs": 4359, + "startTime": "2026-10-01T22:09:22.804Z", + "error": null, + "measurements": [ + { + "kind": "start", + "data": { + "at": "2026-10-01T22:09:22.818Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2451848 kB", + "MemAvailable: 2480472 kB", + "SwapFree: 0 kB", + "Shmem: 4021684 kB" + ], + "fileNr": "288\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 8513, + "kind": "network", + "fdCount": 20, + "status": [ + "Name:\tchrome-headless", + "PPid:\t8492", + "FDSize:\t128", + "VmSize:\t50617288 kB", + "VmRSS:\t 14788 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 4, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1 + } + } + ] + } + }, + { + "kind": "peak", + "data": { + "maximumFD": 707, + "maximumPending": 265, + "failures": 0, + "sample": { + "at": "2026-10-01T22:09:27.077Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 119560 kB", + "MemAvailable: 153952 kB", + "SwapFree: 0 kB", + "Shmem: 6173108 kB" + ], + "fileNr": "1696\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 8513, + "kind": "network", + "fdCount": 42, + "status": [ + "Name:\tchrome-headless", + "PPid:\t8492", + "FDSize:\t128", + "VmSize:\t50618576 kB", + "VmRSS:\t 21588 kB", + "Threads:\t8" + ] + }, + { + "pid": 8727, + "kind": "renderer", + "fdCount": 707, + "status": [ + "Name:\tchrome-headless", + "PPid:\t8495", + "FDSize:\t1024", + "VmSize:\t1519912192 kB", + "VmRSS:\t 141560 kB", + "Threads:\t14" + ] + }, + { + "pid": 8754, + "kind": "renderer", + "fdCount": 707, + "status": [ + "Name:\tchrome-headless", + "PPid:\t8495", + "FDSize:\t1024", + "VmSize:\t1519959064 kB", + "VmRSS:\t 128964 kB", + "Threads:\t20" + ] + } + ] + } + } + }, + { + "kind": "end", + "data": { + "at": "2026-10-01T22:09:27.160Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 341484 kB", + "MemAvailable: 372820 kB", + "SwapFree: 0 kB", + "Shmem: 6159348 kB" + ], + "fileNr": "1024\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 8513, + "kind": "network", + "fdCount": 30, + "status": [ + "Name:\tchrome-headless", + "PPid:\t8492", + "FDSize:\t128", + "VmSize:\t50617552 kB", + "VmRSS:\t 21476 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 12, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 2 + } + }, + { + "pid": 8727, + "kind": "renderer", + "fdCount": 707, + "status": [ + "Name:\tchrome-headless", + "PPid:\t8495", + "FDSize:\t1024", + "VmSize:\t1519912528 kB", + "VmRSS:\t 142856 kB", + "Threads:\t14" + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "shared-memory": 677, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4 + } + } + ] + } + } + ], + "failedRequestCount": 0, + "firstFailedRequests": [] + } + ] + }, + { + "experiment": 3, + "runDirectory": "/home/user/workspace/wt/television/investigate-chromium-insufficient-resources/.test-runs/2026-10-01T22-10-17-150Z-p2876640-r6c20ebd19d66a71a", + "commit": "0ba2dcdaed7c9a2d7692be012255d4b5ceb7d5af", + "sandbox": "tv-testshard-x64-poc-15", + "result": { + "id": "2026-10-01T22-10-17-150Z-p2876640-r6c20ebd19d66a71a", + "commandProvider": "blaxel", + "timingProvider": "blaxel-playwright-x64-4vcpu", + "status": "failed", + "infraStatus": "completed", + "testStatus": "failed", + "startedAt": "2026-10-01T22:10:18.844Z", + "completedAt": "2026-10-01T22:15:08.685Z", + "durationMs": 289841 + }, + "attempts": [ + { + "title": "executable theme reset reloads connected Chromium documents under confirmed state", + "retry": 0, + "status": "timedOut", + "durationMs": 90053, + "startTime": "2026-10-01T22:11:33.511Z", + "error": "\u001b[31mTest timeout of 90000ms exceeded.\u001b[39m", + "measurements": [ + { + "kind": "start", + "data": { + "at": "2026-10-01T22:11:33.527Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 1639032 kB", + "MemAvailable: 1709856 kB", + "SwapFree: 0 kB", + "Shmem: 4582788 kB" + ], + "fileNr": "288\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 6507, + "kind": "network", + "fdCount": 20, + "status": [ + "Name:\tchrome-headless", + "PPid:\t6487", + "FDSize:\t128", + "VmSize:\t50617288 kB", + "VmRSS:\t 15032 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 4, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1 + } + } + ] + } + }, + { + "kind": "peak", + "data": { + "maximumFD": 707, + "maximumPending": 264, + "failures": 0, + "sample": { + "at": "2026-10-01T22:11:38.290Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 920584 kB", + "MemAvailable: 862864 kB", + "SwapFree: 0 kB", + "Shmem: 5410524 kB" + ], + "fileNr": "1024\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 6507, + "kind": "network", + "fdCount": 34, + "status": [ + "Name:\tchrome-headless", + "PPid:\t6487", + "FDSize:\t128", + "VmSize:\t50617932 kB", + "VmRSS:\t 20804 kB", + "Threads:\t8" + ] + }, + { + "pid": 6742, + "kind": "renderer", + "fdCount": 707, + "status": [ + "Name:\tchrome-headless", + "PPid:\t6490", + "FDSize:\t1024", + "VmSize:\t1519918084 kB", + "VmRSS:\t 128412 kB", + "Threads:\t15" + ] + } + ] + } + } + }, + { + "kind": "end", + "data": { + "at": "2026-10-01T22:13:03.553Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2545440 kB", + "MemAvailable: 2485900 kB", + "SwapFree: 0 kB", + "Shmem: 4057056 kB" + ], + "fileNr": "320\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 6507, + "kind": "network", + "fdCount": 28, + "status": [ + "Name:\tchrome-headless", + "PPid:\t6487", + "FDSize:\t128", + "VmSize:\t50617420 kB", + "VmRSS:\t 18732 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 11, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 1 + } + } + ] + } + } + ], + "failedRequestCount": 0, + "firstFailedRequests": [] + }, + { + "title": "executable theme reset reloads connected Chromium documents under confirmed state", + "retry": 1, + "status": "timedOut", + "durationMs": 90325, + "startTime": "2026-10-01T22:13:04.817Z", + "error": "\u001b[31mTest timeout of 90000ms exceeded.\u001b[39m", + "measurements": [ + { + "kind": "start", + "data": { + "at": "2026-10-01T22:13:04.943Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2884208 kB", + "MemAvailable: 2824140 kB", + "SwapFree: 0 kB", + "Shmem: 4043640 kB" + ], + "fileNr": "224\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 6896, + "kind": "network", + "fdCount": 20, + "status": [ + "Name:\tchrome-headless", + "PPid:\t6876", + "FDSize:\t128", + "VmSize:\t50617256 kB", + "VmRSS:\t 8684 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 4, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1 + } + } + ] + } + }, + { + "kind": "peak", + "data": { + "maximumFD": 707, + "maximumPending": 264, + "failures": 0, + "sample": { + "at": "2026-10-01T22:13:10.753Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 999608 kB", + "MemAvailable: 941196 kB", + "SwapFree: 0 kB", + "Shmem: 5450248 kB" + ], + "fileNr": "1024\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 6896, + "kind": "network", + "fdCount": 34, + "status": [ + "Name:\tchrome-headless", + "PPid:\t6876", + "FDSize:\t128", + "VmSize:\t50617932 kB", + "VmRSS:\t 20588 kB", + "Threads:\t8" + ] + }, + { + "pid": 6957, + "kind": "renderer", + "fdCount": 707, + "status": [ + "Name:\tchrome-headless", + "PPid:\t6879", + "FDSize:\t1024", + "VmSize:\t1519918080 kB", + "VmRSS:\t 139016 kB", + "Threads:\t15" + ] + } + ] + } + } + }, + { + "kind": "end", + "data": { + "at": "2026-10-01T22:14:34.964Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2743300 kB", + "MemAvailable: 2683412 kB", + "SwapFree: 0 kB", + "Shmem: 4096916 kB" + ], + "fileNr": "352\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 6896, + "kind": "network", + "fdCount": 28, + "status": [ + "Name:\tchrome-headless", + "PPid:\t6876", + "FDSize:\t128", + "VmSize:\t50617420 kB", + "VmRSS:\t 18272 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 11, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 1 + } + } + ] + } + } + ], + "failedRequestCount": 0, + "firstFailedRequests": [] + }, + { + "title": "executable theme reset reloads connected Chromium documents under confirmed state", + "retry": 2, + "status": "failed", + "durationMs": 9071, + "startTime": "2026-10-01T22:14:36.535Z", + "error": "Error: page.evaluate: Error: Application root did not render an expected state; last state: missing\n at eval (eval at evaluate (:290:30), :32:14)", + "measurements": [ + { + "kind": "start", + "data": { + "at": "2026-10-01T22:14:36.653Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3072172 kB", + "MemAvailable: 3012020 kB", + "SwapFree: 0 kB", + "Shmem: 4072512 kB" + ], + "fileNr": "192\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 7105, + "kind": "network", + "fdCount": 20, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7083", + "FDSize:\t128", + "VmSize:\t50617288 kB", + "VmRSS:\t 8816 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 4, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1 + } + } + ] + } + }, + { + "kind": "failure-resources", + "data": { + "at": "2026-10-01T22:14:41.120Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 127620 kB", + "MemAvailable: 70980 kB", + "SwapFree: 0 kB", + "Shmem: 6518872 kB" + ], + "fileNr": "1536\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 7105, + "kind": "network", + "fdCount": 36, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7083", + "FDSize:\t128", + "VmSize:\t50618064 kB", + "VmRSS:\t 25240 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 14, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 6 + } + }, + { + "pid": 7128, + "kind": "renderer", + "fdCount": 694, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7086", + "FDSize:\t1024", + "VmSize:\t1519897128 kB", + "VmRSS:\t 132712 kB", + "Threads:\t15" + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "shared-memory": 664, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4 + } + }, + { + "pid": 7165, + "kind": "renderer", + "fdCount": 563, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7086", + "FDSize:\t1024", + "VmSize:\t1519618820 kB", + "VmRSS:\t 114564 kB", + "Threads:\t14" + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "shared-memory": 533, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4 + } + } + ] + } + }, + { + "kind": "peak", + "data": { + "maximumFD": 694, + "maximumPending": 226, + "failures": 136, + "sample": { + "at": "2026-10-01T22:14:41.179Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 150892 kB", + "MemAvailable: 93792 kB", + "SwapFree: 0 kB", + "Shmem: 6519052 kB" + ], + "fileNr": "1536\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 7105, + "kind": "network", + "fdCount": 37, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7083", + "FDSize:\t128", + "VmSize:\t50618064 kB", + "VmRSS:\t 24680 kB", + "Threads:\t8" + ] + }, + { + "pid": 7128, + "kind": "renderer", + "fdCount": 694, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7086", + "FDSize:\t1024", + "VmSize:\t1519897128 kB", + "VmRSS:\t 132712 kB", + "Threads:\t15" + ] + }, + { + "pid": 7165, + "kind": "renderer", + "fdCount": 563, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7086", + "FDSize:\t1024", + "VmSize:\t1519618820 kB", + "VmRSS:\t 113996 kB", + "Threads:\t14" + ] + } + ] + } + } + }, + { + "kind": "end", + "data": { + "at": "2026-10-01T22:14:45.682Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 374264 kB", + "MemAvailable: 317216 kB", + "SwapFree: 0 kB", + "Shmem: 6385496 kB" + ], + "fileNr": "992\t0\t814457", + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 7105, + "kind": "network", + "fdCount": 28, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7083", + "FDSize:\t128", + "VmSize:\t50617420 kB", + "VmRSS:\t 20116 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 11, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 1 + } + }, + { + "pid": 7128, + "kind": "renderer", + "fdCount": 694, + "status": [ + "Name:\tchrome-headless", + "PPid:\t7086", + "FDSize:\t1024", + "VmSize:\t1519889096 kB", + "VmRSS:\t 124724 kB", + "Threads:\t15" + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "shared-memory": 664, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4 + } + } + ] + } + } + ], + "failedRequestCount": 136, + "firstFailedRequests": [ + { + "url": "http://127.0.0.1:39101/packages/artifact/src/missing-artifact-page.ts", + "pending": 136, + "maximumPending": 226 + }, + { + "url": "http://127.0.0.1:39101/packages/artifact/src/link-target.ts", + "pending": 135, + "maximumPending": 226 + }, + { + "url": "http://127.0.0.1:39101/packages/artifact/src/index.ts", + "pending": 134, + "maximumPending": 226 + } + ] + } + ] + }, + { + "experiment": 4, + "runDirectory": "/home/user/workspace/wt/television/investigate-chromium-insufficient-resources/.test-runs/2026-10-01T22-14-34-254Z-p2879064-rb0056a09a8053d4b", + "commit": "8a75e3a9d2d59e3e3b8ffb6550aa05a79b8b7d2b", + "sandbox": "tv-testshard-x64-poc-32", + "result": { + "id": "2026-10-01T22-14-34-254Z-p2879064-rb0056a09a8053d4b", + "commandProvider": "blaxel", + "timingProvider": "blaxel-playwright-x64-4vcpu", + "status": "failed", + "infraStatus": "completed", + "testStatus": "failed", + "startedAt": "2026-10-01T22:14:35.874Z", + "completedAt": "2026-10-01T22:19:49.221Z", + "durationMs": 313347 + }, + "attempts": [ + { + "title": "executable theme reset reloads connected Chromium documents under confirmed state", + "retry": 0, + "status": "timedOut", + "durationMs": 90073, + "startTime": "2026-10-01T22:15:14.179Z", + "error": "\u001b[31mTest timeout of 90000ms exceeded.\u001b[39m", + "measurements": [ + { + "kind": "start", + "data": { + "at": "2026-10-01T22:15:14.513Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2503260 kB", + "MemAvailable: 2577740 kB", + "SwapFree: 0 kB", + "Shmem: 4181444 kB" + ], + "fileNr": "192\t0\t814457", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 2394783744 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172300288 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 1826, + "kind": "network", + "fdCount": 20, + "status": [ + "Name:\tchrome-headless", + "PPid:\t1805", + "FDSize:\t128", + "VmSize:\t50617288 kB", + "VmRSS:\t 8788 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 4, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1 + }, + "sharedMemorySizes": {}, + "sharedMemoryAllocated": 0 + } + ] + } + }, + { + "kind": "peak", + "data": { + "maximumFD": 707, + "maximumPending": 259, + "failures": 0, + "sample": { + "at": "2026-10-01T22:15:20.351Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 980656 kB", + "MemAvailable: 926220 kB", + "SwapFree: 0 kB", + "Shmem: 5575128 kB" + ], + "fileNr": "992\t0\t814457", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 967827456 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172300288 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 1826, + "kind": "network", + "fdCount": 34, + "status": [ + "Name:\tchrome-headless", + "PPid:\t1805", + "FDSize:\t128", + "VmSize:\t50617932 kB", + "VmRSS:\t 18420 kB", + "Threads:\t8" + ] + }, + { + "pid": 1888, + "kind": "renderer", + "fdCount": 707, + "status": [ + "Name:\tchrome-headless", + "PPid:\t1808", + "FDSize:\t1024", + "VmSize:\t1519918084 kB", + "VmRSS:\t 128300 kB", + "Threads:\t15" + ] + } + ] + } + } + }, + { + "kind": "lowest-resources", + "data": { + "minimumDiskAvailable": 279891968, + "minimumMemoryAvailableKB": 181372, + "sample": { + "at": "2026-10-01T22:15:19.853Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 136628 kB", + "MemAvailable: 181372 kB", + "SwapFree: 0 kB", + "Shmem: 6246452 kB" + ], + "fileNr": "1344\t0\t814457", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 279891968 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172300288 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 1826, + "kind": "network", + "fdCount": 36, + "status": [ + "Name:\tchrome-headless", + "PPid:\t1805", + "FDSize:\t128", + "VmSize:\t50618064 kB", + "VmRSS:\t 24920 kB", + "Threads:\t8" + ] + }, + { + "pid": 1850, + "kind": "renderer", + "fdCount": 548, + "status": [ + "Name:\tchrome-headless", + "PPid:\t1808", + "FDSize:\t1024", + "VmSize:\t1519598596 kB", + "VmRSS:\t 134136 kB", + "Threads:\t15" + ] + }, + { + "pid": 1888, + "kind": "renderer", + "fdCount": 524, + "status": [ + "Name:\tchrome-headless", + "PPid:\t1808", + "FDSize:\t1024", + "VmSize:\t1519547140 kB", + "VmRSS:\t 121612 kB", + "Threads:\t15" + ] + } + ] + } + } + }, + { + "kind": "end", + "data": { + "at": "2026-10-01T22:16:44.525Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2484596 kB", + "MemAvailable: 2428152 kB", + "SwapFree: 0 kB", + "Shmem: 4221796 kB" + ], + "fileNr": "320\t0\t814457", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 2353655808 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172300288 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 1826, + "kind": "network", + "fdCount": 28, + "status": [ + "Name:\tchrome-headless", + "PPid:\t1805", + "FDSize:\t128", + "VmSize:\t50617420 kB", + "VmRSS:\t 16712 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 11, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 1 + }, + "sharedMemorySizes": { + "16": 1 + }, + "sharedMemoryAllocated": 4096 + } + ] + } + } + ], + "failedRequestCount": 0, + "firstFailedRequests": [] + }, + { + "title": "executable theme reset reloads connected Chromium documents under confirmed state", + "retry": 1, + "status": "timedOut", + "durationMs": 90396, + "startTime": "2026-10-01T22:16:45.952Z", + "error": "\u001b[31mTest timeout of 90000ms exceeded.\u001b[39m", + "measurements": [ + { + "kind": "start", + "data": { + "at": "2026-10-01T22:16:46.118Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2728680 kB", + "MemAvailable: 2671804 kB", + "SwapFree: 0 kB", + "Shmem: 4208548 kB" + ], + "fileNr": "256\t0\t814457", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 2367131648 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172300288 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 2047, + "kind": "network", + "fdCount": 20, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2020", + "FDSize:\t128", + "VmSize:\t50617244 kB", + "VmRSS:\t 8284 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 4, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1 + }, + "sharedMemorySizes": {}, + "sharedMemoryAllocated": 0 + } + ] + } + }, + { + "kind": "peak", + "data": { + "maximumFD": 707, + "maximumPending": 266, + "failures": 0, + "sample": { + "at": "2026-10-01T22:16:51.998Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 851524 kB", + "MemAvailable: 796264 kB", + "SwapFree: 0 kB", + "Shmem: 5615176 kB" + ], + "fileNr": "992\t0\t814457", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 926785536 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172300288 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 2047, + "kind": "network", + "fdCount": 34, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2020", + "FDSize:\t128", + "VmSize:\t50617932 kB", + "VmRSS:\t 19272 kB", + "Threads:\t8" + ] + }, + { + "pid": 2102, + "kind": "renderer", + "fdCount": 707, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2023", + "FDSize:\t1024", + "VmSize:\t1519918676 kB", + "VmRSS:\t 139908 kB", + "Threads:\t15" + ] + } + ] + } + } + }, + { + "kind": "lowest-resources", + "data": { + "minimumDiskAvailable": 267485184, + "minimumMemoryAvailableKB": 69600, + "sample": { + "at": "2026-10-01T22:16:51.751Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 123212 kB", + "MemAvailable: 69600 kB", + "SwapFree: 0 kB", + "Shmem: 6258792 kB" + ], + "fileNr": "1312\t0\t814457", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 267485184 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172300288 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 2047, + "kind": "network", + "fdCount": 36, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2020", + "FDSize:\t128", + "VmSize:\t50618064 kB", + "VmRSS:\t 24616 kB", + "Threads:\t8" + ] + }, + { + "pid": 2065, + "kind": "renderer", + "fdCount": 362, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2023", + "FDSize:\t512", + "VmSize:\t1519208992 kB", + "VmRSS:\t 132004 kB", + "Threads:\t14" + ] + }, + { + "pid": 2102, + "kind": "renderer", + "fdCount": 696, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2023", + "FDSize:\t1024", + "VmSize:\t1519899396 kB", + "VmRSS:\t 121356 kB", + "Threads:\t15" + ] + } + ] + } + } + }, + { + "kind": "end", + "data": { + "at": "2026-10-01T22:18:16.137Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2540924 kB", + "MemAvailable: 2484324 kB", + "SwapFree: 0 kB", + "Shmem: 4261040 kB" + ], + "fileNr": "352\t0\t814457", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 2313469952 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172300288 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 2047, + "kind": "network", + "fdCount": 28, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2020", + "FDSize:\t128", + "VmSize:\t50617420 kB", + "VmRSS:\t 17128 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 11, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 1 + }, + "sharedMemorySizes": { + "16": 1 + }, + "sharedMemoryAllocated": 4096 + } + ] + } + } + ], + "failedRequestCount": 0, + "firstFailedRequests": [] + }, + { + "title": "executable theme reset reloads connected Chromium documents under confirmed state", + "retry": 2, + "status": "timedOut", + "durationMs": 90068, + "startTime": "2026-10-01T22:18:17.974Z", + "error": "\u001b[31mTest timeout of 90000ms exceeded.\u001b[39m", + "measurements": [ + { + "kind": "start", + "data": { + "at": "2026-10-01T22:18:18.103Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2860164 kB", + "MemAvailable: 2803208 kB", + "SwapFree: 0 kB", + "Shmem: 4237056 kB" + ], + "fileNr": "224\t0\t814457", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 2337763328 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172300288 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 2254, + "kind": "network", + "fdCount": 19, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2232", + "FDSize:\t128", + "VmSize:\t50617244 kB", + "VmRSS:\t 8292 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 8, + "socket": 4, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1 + }, + "sharedMemorySizes": {}, + "sharedMemoryAllocated": 0 + } + ] + } + }, + { + "kind": "peak", + "data": { + "maximumFD": 707, + "maximumPending": 267, + "failures": 0, + "sample": { + "at": "2026-10-01T22:18:23.193Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 1069824 kB", + "MemAvailable: 1014404 kB", + "SwapFree: 0 kB", + "Shmem: 5631696 kB" + ], + "fileNr": "1024\t0\t814457", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 909901824 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172300288 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 2254, + "kind": "network", + "fdCount": 34, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2232", + "FDSize:\t128", + "VmSize:\t50617932 kB", + "VmRSS:\t 19736 kB", + "Threads:\t8" + ] + }, + { + "pid": 2313, + "kind": "renderer", + "fdCount": 707, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2235", + "FDSize:\t1024", + "VmSize:\t1519910148 kB", + "VmRSS:\t 127852 kB", + "Threads:\t14" + ] + } + ] + } + } + }, + { + "kind": "lowest-resources", + "data": { + "minimumDiskAvailable": 58142720, + "minimumMemoryAvailableKB": 84688, + "sample": { + "at": "2026-10-01T22:18:22.698Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 138392 kB", + "MemAvailable: 84688 kB", + "SwapFree: 0 kB", + "Shmem: 6463200 kB" + ], + "fileNr": "1440\t0\t814457", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 58142720 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172300288 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 2254, + "kind": "network", + "fdCount": 36, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2232", + "FDSize:\t128", + "VmSize:\t50618064 kB", + "VmRSS:\t 23876 kB", + "Threads:\t8" + ] + }, + { + "pid": 2277, + "kind": "renderer", + "fdCount": 606, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2235", + "FDSize:\t1024", + "VmSize:\t1519717384 kB", + "VmRSS:\t 135244 kB", + "Threads:\t15" + ] + }, + { + "pid": 2313, + "kind": "renderer", + "fdCount": 544, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2235", + "FDSize:\t1024", + "VmSize:\t1519579908 kB", + "VmRSS:\t 121728 kB", + "Threads:\t14" + ] + } + ] + } + } + }, + { + "kind": "end", + "data": { + "at": "2026-10-01T22:19:48.124Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2613900 kB", + "MemAvailable: 2557356 kB", + "SwapFree: 0 kB", + "Shmem: 4278036 kB" + ], + "fileNr": "352\t0\t814457", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 2296057856 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172300288 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 2254, + "kind": "network", + "fdCount": 28, + "status": [ + "Name:\tchrome-headless", + "PPid:\t2232", + "FDSize:\t128", + "VmSize:\t50617420 kB", + "VmRSS:\t 17824 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 11, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 1 + }, + "sharedMemorySizes": { + "16": 1 + }, + "sharedMemoryAllocated": 4096 + } + ] + } + } + ], + "failedRequestCount": 0, + "firstFailedRequests": [] + } + ] + }, + { + "experiment": 5, + "runDirectory": "/home/user/workspace/wt/television/investigate-chromium-insufficient-resources/.test-runs/2026-10-01T22-21-04-731Z-p2882965-r4564bb4b4668cd6d", + "commit": "9743dbe7b734adbe3e0a6658f8f6c7611ef49389", + "sandbox": "tv-testshard-x64-poc-03", + "result": { + "id": "2026-10-01T22-21-04-731Z-p2882965-r4564bb4b4668cd6d", + "commandProvider": "blaxel", + "timingProvider": "blaxel-playwright-x64-4vcpu", + "status": "passed", + "infraStatus": "completed", + "testStatus": "passed", + "startedAt": "2026-10-01T22:21:06.332Z", + "completedAt": "2026-10-01T22:21:34.890Z", + "durationMs": 28558 + }, + "attempts": [ + { + "title": "executable theme reset reloads connected Chromium documents under confirmed state", + "retry": 0, + "status": "passed", + "durationMs": 7021, + "startTime": "2026-10-01T22:21:22.913Z", + "error": null, + "measurements": [ + { + "kind": "start", + "data": { + "at": "2026-10-01T22:21:23.157Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3946536 kB", + "MemAvailable: 3936572 kB", + "SwapFree: 0 kB", + "Shmem: 2877360 kB" + ], + "fileNr": "224\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 3729829888 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 11666, + "kind": "network", + "fdCount": 11, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11646", + "FDSize:\t128", + "VmSize:\t50559728 kB", + "VmRSS:\t 7512 kB", + "Threads:\t1" + ], + "descriptorKinds": { + "file": 8, + "socket": 3 + }, + "sharedMemorySizes": {}, + "sharedMemoryAllocated": 0 + } + ] + } + }, + { + "kind": "ROSE_GC_BEFORE", + "data": { + "at": "2026-10-01T22:21:26.951Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2730892 kB", + "MemAvailable: 2724816 kB", + "SwapFree: 0 kB", + "Shmem: 3822512 kB" + ], + "fileNr": "832\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 2762153984 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 11666, + "kind": "network", + "fdCount": 42, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11646", + "FDSize:\t128", + "VmSize:\t50618576 kB", + "VmRSS:\t 19800 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 16, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 10 + }, + "sharedMemorySizes": { + "16": 2, + "131000": 8 + }, + "sharedMemoryAllocated": 1056768 + }, + { + "pid": 11694, + "kind": "renderer", + "fdCount": 259, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1519017284 kB", + "VmRSS:\t 94780 kB", + "Threads:\t15" + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "shared-memory": 229, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4 + }, + "sharedMemorySizes": { + "131000": 8, + "2097152": 220, + "4198400": 1 + }, + "sharedMemoryAllocated": 466620416 + }, + { + "pid": 11730, + "kind": "renderer", + "fdCount": 259, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1519015940 kB", + "VmRSS:\t 86480 kB", + "Threads:\t15" + ], + "descriptorKinds": { + "file": 11, + "socket": 11, + "shared-memory": 229, + "anon_inode:[eventfd]": 4, + "anon_inode:[eventpoll]": 4 + }, + "sharedMemorySizes": { + "131000": 8, + "2097152": 220, + "4198400": 1 + }, + "sharedMemoryAllocated": 466620416 + } + ] + } + }, + { + "kind": "ROSE_GC_AFTER", + "data": { + "at": "2026-10-01T22:21:27.072Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3699996 kB", + "MemAvailable: 3692256 kB", + "SwapFree: 0 kB", + "Shmem: 2920608 kB" + ], + "fileNr": "416\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 3685933056 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 11666, + "kind": "network", + "fdCount": 42, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11646", + "FDSize:\t128", + "VmSize:\t50618576 kB", + "VmRSS:\t 19800 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 16, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 10 + }, + "sharedMemorySizes": { + "16": 2, + "131000": 8 + }, + "sharedMemoryAllocated": 1056768 + } + ] + } + }, + { + "kind": "ROSE_GC_BEFORE", + "data": { + "at": "2026-10-01T22:21:28.387Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3506724 kB", + "MemAvailable: 3498712 kB", + "SwapFree: 0 kB", + "Shmem: 2944532 kB" + ], + "fileNr": "448\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 3661377536 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 11666, + "kind": "network", + "fdCount": 42, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11646", + "FDSize:\t128", + "VmSize:\t50618576 kB", + "VmRSS:\t 19992 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 16, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 10 + }, + "sharedMemorySizes": { + "16": 2, + "131000": 8 + }, + "sharedMemoryAllocated": 1056768 + } + ] + } + }, + { + "kind": "ROSE_GC_AFTER", + "data": { + "at": "2026-10-01T22:21:28.498Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3542992 kB", + "MemAvailable: 3534980 kB", + "SwapFree: 0 kB", + "Shmem: 2927288 kB" + ], + "fileNr": "416\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 3679182848 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 11666, + "kind": "network", + "fdCount": 42, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11646", + "FDSize:\t128", + "VmSize:\t50618576 kB", + "VmRSS:\t 19992 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 16, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 10 + }, + "sharedMemorySizes": { + "16": 2, + "131000": 8 + }, + "sharedMemoryAllocated": 1056768 + } + ] + } + }, + { + "kind": "ROSE_GC_BEFORE", + "data": { + "at": "2026-10-01T22:21:29.888Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3332724 kB", + "MemAvailable: 3324684 kB", + "SwapFree: 0 kB", + "Shmem: 2951136 kB" + ], + "fileNr": "448\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 3654643712 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 11666, + "kind": "network", + "fdCount": 42, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11646", + "FDSize:\t128", + "VmSize:\t50618576 kB", + "VmRSS:\t 20172 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 16, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 10 + }, + "sharedMemorySizes": { + "16": 2, + "131000": 8 + }, + "sharedMemoryAllocated": 1056768 + } + ] + } + }, + { + "kind": "ROSE_GC_AFTER", + "data": { + "at": "2026-10-01T22:21:29.953Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3341408 kB", + "MemAvailable: 3333368 kB", + "SwapFree: 0 kB", + "Shmem: 2933852 kB" + ], + "fileNr": "448\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 3672453120 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 11666, + "kind": "network", + "fdCount": 42, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11646", + "FDSize:\t128", + "VmSize:\t50618576 kB", + "VmRSS:\t 20172 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 16, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 10 + }, + "sharedMemorySizes": { + "16": 2, + "131000": 8 + }, + "sharedMemoryAllocated": 1056768 + } + ] + } + }, + { + "kind": "peak", + "data": { + "maximumFD": 255, + "maximumPending": 268, + "failures": 0, + "sample": { + "at": "2026-10-01T22:21:26.834Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2772672 kB", + "MemAvailable: 2766592 kB", + "SwapFree: 0 kB", + "Shmem: 3806180 kB" + ], + "fileNr": "832\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 2779287552 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 11666, + "kind": "network", + "fdCount": 42, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11646", + "FDSize:\t128", + "VmSize:\t50618576 kB", + "VmRSS:\t 19796 kB", + "Threads:\t8" + ] + }, + { + "pid": 11694, + "kind": "renderer", + "fdCount": 255, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1519009092 kB", + "VmRSS:\t 91164 kB", + "Threads:\t15" + ] + }, + { + "pid": 11730, + "kind": "renderer", + "fdCount": 255, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1519007748 kB", + "VmRSS:\t 83836 kB", + "Threads:\t15" + ] + } + ] + } + } + }, + { + "kind": "lowest-resources", + "data": { + "minimumDiskAvailable": 2779287552, + "minimumMemoryAvailableKB": 2766592, + "sample": { + "at": "2026-10-01T22:21:26.834Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 2772672 kB", + "MemAvailable: 2766592 kB", + "SwapFree: 0 kB", + "Shmem: 3806180 kB" + ], + "fileNr": "832\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 2779287552 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 11666, + "kind": "network", + "fdCount": 42, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11646", + "FDSize:\t128", + "VmSize:\t50618576 kB", + "VmRSS:\t 19796 kB", + "Threads:\t8" + ] + }, + { + "pid": 11694, + "kind": "renderer", + "fdCount": 255, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1519009092 kB", + "VmRSS:\t 91164 kB", + "Threads:\t15" + ] + }, + { + "pid": 11730, + "kind": "renderer", + "fdCount": 255, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11649", + "FDSize:\t512", + "VmSize:\t1519007748 kB", + "VmRSS:\t 83836 kB", + "Threads:\t15" + ] + } + ] + } + } + }, + { + "kind": "end", + "data": { + "at": "2026-10-01T22:21:30.129Z", + "memory": [ + "MemTotal: 8149540 kB", + "MemFree: 3415728 kB", + "MemAvailable: 3407672 kB", + "SwapFree: 0 kB", + "Shmem: 2922468 kB" + ], + "fileNr": "384\t0\t814457", + "oomKills": "oom_kill 0", + "filesystems": { + "/tmp": { + "total": 6676103168, + "available": 3683971072 + }, + "/dev/shm": { + "total": 4172562432, + "available": 4172562432 + } + }, + "cgroup": { + "memory.current": "unavailable", + "memory.max": "unavailable", + "memory.events": "unavailable", + "pids.current": "unavailable", + "pids.max": "unavailable", + "cpu.stat": "unavailable" + }, + "processes": [ + { + "pid": 11666, + "kind": "network", + "fdCount": 28, + "status": [ + "Name:\tchrome-headless", + "PPid:\t11646", + "FDSize:\t128", + "VmSize:\t50617420 kB", + "VmRSS:\t 19656 kB", + "Threads:\t8" + ], + "descriptorKinds": { + "file": 9, + "socket": 11, + "anon_inode:[eventfd]": 3, + "anon_inode:[eventpoll]": 3, + "anon_inode:inotify": 1, + "shared-memory": 1 + }, + "sharedMemorySizes": { + "16": 1 + }, + "sharedMemoryAllocated": 4096 + } + ] + } + } + ], + "failedRequestCount": 0, + "firstFailedRequests": [] + } + ] + } +] diff --git a/docs/working/chromium-insufficient-resources-evidence/run3-kernel-oom.txt b/docs/working/chromium-insufficient-resources-evidence/run3-kernel-oom.txt new file mode 100644 index 00000000..a59317f9 --- /dev/null +++ b/docs/working/chromium-insufficient-resources-evidence/run3-kernel-oom.txt @@ -0,0 +1,8 @@ +[14494.236116] Chrome_ChildIOT invoked oom-killer: gfp_mask=0x140cca(GFP_HIGHUSER_MOVABLE|__GFP_COMP), order=0, oom_score_adj=0 +[14494.238238] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=chrome-headless,pid=6721,uid=0 +[14494.238317] Out of memory: Killed process 6721 (chrome-headless) total-vm:1519582212kB, anon-rss:123652kB, file-rss:0kB, shmem-rss:9544kB, UID:0 pgtables:1020kB oom_score_adj:300 +[14586.820194] libuv-worker invoked oom-killer: gfp_mask=0x140cca(GFP_HIGHUSER_MOVABLE|__GFP_COMP), order=0, oom_score_adj=0 +[14586.820722] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=chrome-headless,pid=6921,uid=0 +[14586.820769] Out of memory: Killed process 6921 (chrome-headless) total-vm:1519461376kB, anon-rss:122196kB, file-rss:0kB, shmem-rss:9892kB, UID:0 pgtables:1052kB oom_score_adj:300 +14741.33 45998.36 +Thu Oct 1 22:20:48 UTC 2026 diff --git a/docs/working/chromium-insufficient-resources-evidence/run4-kernel-oom.txt b/docs/working/chromium-insufficient-resources-evidence/run4-kernel-oom.txt new file mode 100644 index 00000000..b20522bd --- /dev/null +++ b/docs/working/chromium-insufficient-resources-evidence/run4-kernel-oom.txt @@ -0,0 +1,11 @@ +[20373.019890] Chrome_ChildIOT invoked oom-killer: gfp_mask=0x140cca(GFP_HIGHUSER_MOVABLE|__GFP_COMP), order=0, oom_score_adj=0 +[20373.021712] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=chrome-headless,pid=1850,uid=0 +[20373.021763] Out of memory: Killed process 1850 (chrome-headless) total-vm:1519621124kB, anon-rss:124324kB, file-rss:0kB, shmem-rss:9596kB, UID:0 pgtables:1040kB oom_score_adj:300 +[20464.830001] Chrome_ChildIOT invoked oom-killer: gfp_mask=0x140cc0(GFP_USER|__GFP_COMP), order=0, oom_score_adj=0 +[20464.830545] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=chrome-headless,pid=2065,uid=0 +[20464.830607] Out of memory: Killed process 2065 (chrome-headless) total-vm:1519254048kB, anon-rss:122396kB, file-rss:0kB, shmem-rss:9684kB, UID:0 pgtables:1048kB oom_score_adj:300 +[20555.798387] Chrome_ChildIOT invoked oom-killer: gfp_mask=0x140cc0(GFP_USER|__GFP_COMP), order=0, oom_score_adj=0 +[20555.798866] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=chrome-headless,pid=2277,uid=0 +[20555.798923] Out of memory: Killed process 2277 (chrome-headless) total-vm:1519766536kB, anon-rss:125460kB, file-rss:0kB, shmem-rss:9820kB, UID:0 pgtables:1016kB oom_score_adj:300 +20674.07 66771.89 +Thu Oct 1 22:20:48 UTC 2026 diff --git a/docs/working/chromium-insufficient-resources.md b/docs/working/chromium-insufficient-resources.md new file mode 100644 index 00000000..21898a03 --- /dev/null +++ b/docs/working/chromium-insufficient-resources.md @@ -0,0 +1,107 @@ +# Chromium resource failures on Blaxel + +Investigator: rose, 2026-10-01. Branch: `thopter/investigate-chromium-insufficient-resources`, based on `68b8ab11`. All five experiments ran through the canonical runner on committed, pushed revisions, on Blaxel, with the default two retries. No tests, browsers, application servers, Electron, builds, lint, or type checks ran on the shared development host. + +## Finding + +The resource pressure is **Chromium response-body shared memory competing with a persistent, RAM-backed sandbox filesystem**, with two observed failure modes: failed resource loads and kernel OOM kills of renderers. Neither the entire suite nor another shard is necessary to reproduce it. + +The unbundled Vite application makes hundreds of small module requests. Chromium 148 allocates 2 MiB shared-memory response buffers, many of which remain open after loading and navigation until garbage collection. The two-page theme-reset test loads each document four times. A passing run reached 707 descriptors in each renderer, predominantly shared-memory handles, and consumed several GiB beyond the sandbox's baseline. Forcing garbage collection between reloads released hundreds of these buffers and prevented that accumulation. + +Persistent Blaxel workers already use substantial RAM for their writable filesystem. Playwright launches Chromium with `--disable-dev-shm-usage`, placing shared-memory files under `/tmp` on that filesystem. The root filesystem reported a 6,676,103,168-byte capacity on the 8 GB workers; `/dev/shm` was almost empty. Persisted test runtime directories reduce the space and RAM left for browser buffers. A concrete leak is the cached CLI runtime created by `getSourceCLIEntry()` in `test/helpers/product-server.ts`: it makes a `television-source-cli-e2e-*` directory per worker and never removes it. The original failing sandbox, `poc-131`, had **267 such directories**, with approximately **1.2 GB under `/tmp`** at inspection. + +**Confidence boundary:** RAM exhaustion and renderer OOM kills are directly proven by kernel logs. The `ERR_INSUFFICIENT_RESOURCES` reproduction shows exhausted shared-memory/storage capacity and excludes socket/admission limits for an inspected failed request. The exact failing allocation syscall and errno were not captured; attributing that error specifically to the Mojo response-pipe allocation is a strong source-supported inference, not a syscall trace. + +## Reproduction results + +| Experiment | Revision | Blaxel sandbox | Selection and result | +|---|---|---|---| +| 1 | `0ba2dcda` | `poc-143` | Original theme-reset case plus observation: passed first attempt. Sampled peak: 549,428 KiB available RAM, 5,840,544 KiB shared memory, 707 maximum process descriptors, 261 maximum outstanding requests. | +| 2 | `0ba2dcda` | `poc-39` | Whole theme acceptance file: passed. Theme-reset sampled peak: only 153,952 KiB available RAM, 6,173,108 KiB shared memory, 707 descriptors, 265 outstanding requests. | +| 3 | `0ba2dcda` | `poc-15` | Only original shard 8's assignment: theme reset failed all three attempts. Attempts 0/1 timed out after renderer OOM kills; attempt 2 recorded **136 ERR_INSUFFICIENT_RESOURCES requests**. | +| 4 | `8a75e3a9` | `poc-32` | Original theme-reset case alone, with filesystem measurements: all three attempts timed out. Kernel logs identify **one killed Chromium renderer per attempt**. No request-failed error was required for this failure mode. | +| 5 | `9743dbe7` | `poc-03` | Same case, with diagnostic `page.requestGC()` after each completed reload step: passed first attempt, no OOM kills. First GC released **220 buffers of 2,097,152 bytes in each of two renderers**. Shared memory fell from 3,822,512 to 2,920,608 KiB; root space increased by 923,779,072 bytes. | + +The GC run used a different sandbox with a lower baseline; its pass alone is not a controlled estimate of flake reduction. The within-process disappearance of 440 2 MiB buffers is the causal evidence for retained response buffers. Subsequent reloads stayed near the lower shared-memory level. This is an investigation technique, not a proposed test fix. + +In experiment 3's request-failure snapshot: + +- Available RAM was 70,980 KiB and shared memory was 6,518,872 KiB. +- The two renderers had 694 and 563 descriptors, including 664 and 533 shared-memory descriptors. Every inspected process had a 4,096-descriptor limit. The network service was far below that limit. +- The kernel killed renderer PIDs 6721 and 6921 during attempts 0/1. Their PIDs match the experiment's process records. The final retry reached allocation failures instead. +- Shared memory was within about 1 MiB of the worker's root-filesystem capacity. That instrumentation revision did not record `statfs`, so this is a capacity comparison, not a sampled free-byte count at that error. +- Its NetLog contains eight malformed lines consistent with interrupted writes; do not treat the whole file as valid JSON. Parsing complete lines shows the first failed module request (`missing-artifact-page.ts`, NetLog source 5561) obtained an **HTTP 304 response**, reached `URL_REQUEST_DELEGATE_RESPONSE_STARTED`, and was then cancelled. Its request was admitted and its socket transaction completed. The later failure is not a socket-pool or outstanding-request admission rejection. + +Experiment 4 sampled available RAM down to 69,600 KiB. Kernel logs identify renderer PIDs 1850, 2065, and 2277, corresponding to its three attempts. The minimum sampled `/tmp` free space was 279,891,968, 267,485,184, and 58,142,720 bytes by attempt. Physical memory can exhaust before the filesystem quota; a 250 ms sampler also cannot establish the instantaneous minimum. The absence of ERR_INSUFFICIENT_RESOURCES does not indicate a healthy browser. + +## Original run and shard packing + +Original run: + +`/home/user/workspace/wt/television-desktop-connect-links-plan/.test-runs/2026-10-01T21-41-23-867Z-p2861143-ra3e7bfaa4a5003cd` + +The supplied retry-1 trace, `/tmp/desktop-connect-theme-reset-blaxel-trace.zip`, records Chromium `148.0.7778.96`, 2,017 requests across two pages, four document loads per page, and 16 consecutive SVG-module failures during the fourth load on one page. Each page has approximately 258 unique URLs; total requests must not be mistaken for simultaneous requests. + +Shard 8 runs six unit files, `token-only-persistence.test.ts`, `theme-product-acceptance.test.ts`, then desktop `user-data-identity.test.ts`. Surfaces execute sequentially; theme acceptance uses one Playwright worker. No process leak was reported. Experiment 3 reproduced its failure without the other 35 shards; experiment 4 reproduced OOM in the single case without earlier files. Simultaneous shards, a particular preceding test, and retained processes from earlier files are **not necessary causes**. + +The full-verify correlation is better explained by available capacity on the assigned persistent worker than by a unique full-verify execution mode. Baseline shared memory varied substantially: approximately 2.9 GB on experiment 5's worker versus 4.2 GB on experiment 4's. The original failing worker had only about 2 GB root space free at later inspection. That persisted filesystem is a contributor, not a measurement of exact free space at the historical failure instant. + +## Responsible harness and test behavior + +1. `test/helpers/product-server.ts:getSourceCLIEntry()` creates a temporary build/runtime tree and retains its filename in module state. `disposeAllProductServers()` removes server homes and optional bundled runtime copies, but not the shared source-CLI tree. There is no worker-exit cleanup for it. Blaxel checkout reset cleans the repository and `.testshards` results, not these `/tmp` directories. Worker reuse accumulates hundreds of them. +2. `ProductServer.appURL()` creates a development proxy to Vite. Tests navigating to `/packages/web/src/index.html` load the source-module graph. Theme reset repeatedly reloads two documents together, making buffer accumulation unusually large. The preceding executable-theme acceptance also uses two development pages, but its buffers need not survive into the reset test to cause the failure. +3. Playwright's default `--disable-dev-shm-usage` couples response buffers to the writable root filesystem. The mount reports `upperdir=/mnt/tmp/upper`; writable files contribute to shared-memory pressure. `/dev/shm` stays effectively unused. Moving allocations there would not create more physical RAM, and has not been validated as a fix. +4. A passing process-leak audit cannot detect retained files or predict shared-memory allocation spikes. Browser retries start fresh processes but inherit the same worker's persisted files and limited capacity. + +## Other intermittent tests + +The reported demo-mode case was on **shard 36**, not shard 8. `summary.json` incorrectly points its log to shard 8; the native shard plan and report are the stronger evidence. Its first attempt timed out in `afterEach`, during cleanup of its server/site; retry passed in 1.394 s. This is **not yet attributable** to Chromium resource exhaustion. + +Both `browser-demo-mode.test.ts` and `channel-switcher-context-menu.test.ts` use the unbundled development application; demo-mode also reloads it. They share exposure to the capacity problem, but no trace or kernel evidence was supplied for the channel-switcher drag failure, and this investigation has not established that either intermittent failure shares the cause. Do not relabel all browser flakes as resource failures. + +## Recommended fix and scope + +1. **Fix the test-runtime file leak and reclaim existing accumulation on idle leased workers.** Give the source-CLI runtime a worker lifetime, clean it after its product servers stop, and cover Playwright and other callers of the helper. Preserve once-per-worker build reuse; deleting it after every test would add unnecessary builds. Reclaim demonstrably orphaned generated runtime directories, rather than arbitrary `/tmp` files or processes selected by name. This is test-harness/provider work, with no production behavior change. +2. **Use the built application for product acceptances that do not require development-module behavior.** The PR #9 author's theme-reset change removes the triggering fan-out. Review other `ProductServer.appURL()` consumers for the same distinction; component fixtures that intentionally exercise source modules may need different handling. This is test setup work, not a production bundling change. +3. **Make failures diagnosable.** Retain small per-attempt memory/free-space measurements and OOM-counter changes, plus browser crashes/request errors, when acceptance fails. Process-leak checks alone cannot identify this failure class. Bound artifact retention on persistent workers. + +Increasing retries, globally serializing the suite, or raising descriptor limits is not the primary fix: the failures already occur in one test with one worker and low descriptor counts. More RAM or fresh workers provide capacity, but do not cure runtime-directory retention. Routine forced GC is not the proposed fix. + +There is also a separate reproducibility issue: the coordinator aliases Chromium build 1223 into the build-1208 path expected by Playwright 1.58.2. Its manifest expects Chromium `145.0.7632.6`, while tests actually run `148.0.7778.96`. Provision the matching browser version instead of this alias. Whether version alignment changes buffer retention remains untested; it is not established as the root cause. + +## What remains unknown + +- The exact failed allocation syscall/errno for the original 16 SVG errors. The response-pipe path is inferred from source, accepted HTTP responses, capacity measurements, and the GC experiment. +- Whether Chromium 145 retains the same buffers, and whether Chromium 148's retention is intended GC timing or a regression. +- Whether cleanup alone leaves sufficient headroom for every source-based acceptance on an 8 GB worker. No cleanup-only A/B or post-fix full verify was performed. +- Whether demo-mode teardown or channel-switcher drag flakes share any part of this cause. +- The original worker's exact capacity at the historical failure instant; later inspection cannot recover it. + +## Reproduction and retained evidence + +Experiment revisions remain in branch history. The final report revision restores test/harness files to `68b8ab11`; instrumentation and forced GC are not proposed changes for merge. + +Targeted runs used: + +```sh +npm test -- blaxel --file packages/web/test/e2e/theme-product-acceptance.test.ts --grep 'executable theme reset reloads connected Chromium documents under confirmed state' --ignore-uncommitted --run-dir-output /tmp/rose-chromium-runN +``` + +Experiment 2 omitted `--grep`. Experiment 3 used: + +```sh +npm test -- blaxel --shard-indices 8 --force --ignore-uncommitted --run-dir-output /tmp/rose-chromium-run3 +``` + +`--ignore-uncommitted` acknowledged only a local `node_modules` symlink to already-installed coordinator dependencies. Actual test code was committed and pushed. The first attempt to dispatch experiment 1 stopped at remote preflight because this symlink appeared untracked; it ran no tests. + +[measurements.json](chromium-insufficient-resources-evidence/measurements.json) retains compact measurements, commit/sandbox/run-directory identities, and GC snapshots. Detailed [GC renderer descriptors](chromium-insufficient-resources-evidence/gc-renderers.json) and [original-worker filesystem inspection](chromium-insufficient-resources-evidence/filesystem-inspection.txt) are also retained. Kernel evidence: [shard replay](chromium-insufficient-resources-evidence/run3-kernel-oom.txt), [isolated case](chromium-insufficient-resources-evidence/run4-kernel-oom.txt). Full normalized reports remain under this worktree's `.test-runs/`, with exact paths in the JSON. Downloaded NetLogs remain at `/tmp/rose-run1.netlog.json` and `/tmp/rose-run3-retry2.netlog.json` on the development host. + +Primary Chromium 148 source references: + +- [Response buffer size: 2 MiB](https://github.com/chromium/chromium/blob/148.0.7778.96/services/network/public/cpp/loading_params.cc#L15-L45). +- [Response-pipe allocation failure becomes ERR_INSUFFICIENT_RESOURCES](https://github.com/chromium/chromium/blob/148.0.7778.96/services/network/url_loader.cc#L1216-L1238). +- [DataPipe allocates shared memory](https://github.com/chromium/chromium/blob/148.0.7778.96/mojo/core/ipcz_driver/data_pipe.cc#L145-L170). +- [Shared-memory files and AllocateFileRegion](https://github.com/chromium/chromium/blob/148.0.7778.96/base/memory/platform_shared_memory_region_posix.cc#L189-L243). +- [Outstanding-loader limit: 2,700](https://github.com/chromium/chromium/blob/148.0.7778.96/services/network/network_context.h#L968-L972) and [admission check](https://github.com/chromium/chromium/blob/148.0.7778.96/services/network/url_loader_factory.cc#L238-L283). +- [POSIX descriptor errors also map to this code](https://github.com/chromium/chromium/blob/148.0.7778.96/net/base/net_errors_posix.cc#L84-L113), explaining why the error string alone cannot identify the resource. diff --git a/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03-sizes.json b/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03-sizes.json new file mode 100644 index 00000000..704f61d3 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03-sizes.json @@ -0,0 +1,35 @@ +{ + "sandbox": "tv-testshard-x64-poc-03", + "capturedAt": "2026-10-01T22:46:47.023Z", + "command": "du -B1 --max-depth=1 /tmp", + "sizes": { + "/tmp/node-compile-cache": 8892416, + "/tmp/television-source-cli-e2e-*": 55930880, + "/tmp/television-web-bundle-*": 22622208, + "/tmp/tv-attest-cli-*": 8613888, + "/tmp/tv-attest-prune-*": 8859648, + "/tmp/tv-attest-rundir-*": 24576, + "/tmp/tv-attest-writer-*": 3661824, + "/tmp/tv-attest-clone-*": 393216, + "/tmp/tv-toolchain-npm-*": 8192, + "/tmp/tv-toolchain-policy-mismatch-*": 8192, + "/tmp/tv-toolchain-policy-*": 8192, + "/tmp/tv-test-registry-*": 24576, + "/tmp/tv-provider-reports-*": 12288, + "/tmp/tv-provider-corrupt-*": 4096, + "/tmp/tv-provider-combine-*": 0, + "/tmp/tv-provider-stale-owner-*": 0, + "/tmp/tv-provider-leak-*": 4096, + "/tmp/tv-provider-remote-native-*": 4096, + "/tmp/tv-provider-blaxel-wrapper-*": 4096, + "/tmp/tv-provider-timing-shards-*": 4096, + "/tmp/tv-provider-normalize-*": 8192, + "/tmp/tv-posthog-preflight-missing-*": 0, + "/tmp/tv-posthog-preflight-*": 4096, + "/tmp/.X11-unix": 0, + "/tmp/playwright-transform-cache-0": 19845120, + "/tmp/multipart-uploads": 0, + "/tmp": 156442624 + }, + "exitCode": 0 +} diff --git a/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03.json b/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03.json new file mode 100644 index 00000000..9f3a22f5 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03.json @@ -0,0 +1,420 @@ +{ + "capturedAt": "2026-10-01T22:44:20.524241+00:00", + "tmp": { + "availableBytes": 3700719616, + "totalBytes": 6676103168, + "categories": [ + { + "prefix": ".X*", + "count": 1, + "examples": [ + { + "name": ".X11-unix", + "kind": "dir" + } + ] + }, + { + "prefix": ".org.chromium.Chromium.*", + "count": 9, + "examples": [ + { + "name": ".org.chromium.Chromium.BOuDBO", + "kind": "file" + }, + { + "name": ".org.chromium.Chromium.W2lXu2", + "kind": "file" + } + ] + }, + { + "prefix": "multipart-uploads", + "count": 1, + "examples": [ + { + "name": "multipart-uploads", + "kind": "dir" + } + ] + }, + { + "prefix": "node-compile-cache*", + "count": 1, + "examples": [ + { + "name": "node-compile-cache", + "kind": "dir" + } + ] + }, + { + "prefix": "playwright-*", + "count": 1, + "examples": [ + { + "name": "playwright-transform-cache-0", + "kind": "dir" + } + ] + }, + { + "prefix": "rose-*", + "count": 2, + "examples": [ + { + "name": "rose-chromium-11632.netlog.json", + "kind": "file" + }, + { + "name": "rose-chromium-11632.log", + "kind": "file" + } + ] + }, + { + "prefix": "television-source-cli-e2e-*", + "count": 17, + "examples": [ + { + "name": "television-source-cli-e2e-6bIaZE", + "kind": "dir" + }, + { + "name": "television-source-cli-e2e-P8NxgI", + "kind": "dir" + } + ] + }, + { + "prefix": "television-web-bundle-*", + "count": 27, + "examples": [ + { + "name": "television-web-bundle-0TE0fq", + "kind": "dir" + }, + { + "name": "television-web-bundle-1xECmr", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-attest-cli-*", + "count": 39, + "examples": [ + { + "name": "tv-attest-cli-38qL1c", + "kind": "dir" + }, + { + "name": "tv-attest-cli-6NzZvS", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-attest-clone-*", + "count": 3, + "examples": [ + { + "name": "tv-attest-clone-73s4Yq", + "kind": "dir" + }, + { + "name": "tv-attest-clone-g2egxA", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-attest-prune-*", + "count": 33, + "examples": [ + { + "name": "tv-attest-prune-1Kbfdh", + "kind": "dir" + }, + { + "name": "tv-attest-prune-36WSdp", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-attest-rundir-*", + "count": 3, + "examples": [ + { + "name": "tv-attest-rundir-IudtUg", + "kind": "dir" + }, + { + "name": "tv-attest-rundir-Su49lM", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-attest-writer-*", + "count": 18, + "examples": [ + { + "name": "tv-attest-writer-51Dp9C", + "kind": "dir" + }, + { + "name": "tv-attest-writer-7XgAsh", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-git-askpass.sh", + "count": 1, + "examples": [ + { + "name": "tv-git-askpass.sh", + "kind": "file" + } + ] + }, + { + "prefix": "tv-owner-cleanup-*", + "count": 1, + "examples": [ + { + "name": "tv-owner-cleanup-2026-09-29T18-41-53-559Z-1385612-shard-3.json", + "kind": "file" + } + ] + }, + { + "prefix": "tv-posthog-preflight-*", + "count": 1, + "examples": [ + { + "name": "tv-posthog-preflight-6zpiaH", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-posthog-preflight-missing-*", + "count": 1, + "examples": [ + { + "name": "tv-posthog-preflight-missing-4tzN0y", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-blaxel-wrapper-*", + "count": 1, + "examples": [ + { + "name": "tv-provider-blaxel-wrapper-soo5KV", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-combine-*", + "count": 1, + "examples": [ + { + "name": "tv-provider-combine-XnpJi1", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-corrupt-*", + "count": 1, + "examples": [ + { + "name": "tv-provider-corrupt-3nq4hS", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-leak-*", + "count": 1, + "examples": [ + { + "name": "tv-provider-leak-VXhPIP", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-normalize-*", + "count": 1, + "examples": [ + { + "name": "tv-provider-normalize-Ky71z0", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-remote-native-*", + "count": 1, + "examples": [ + { + "name": "tv-provider-remote-native-H1RqSp", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-reports-*", + "count": 1, + "examples": [ + { + "name": "tv-provider-reports-Y1Nd1C", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-stale-owner-*", + "count": 1, + "examples": [ + { + "name": "tv-provider-stale-owner-9WjHLo", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-timing-shards-*", + "count": 1, + "examples": [ + { + "name": "tv-provider-timing-shards-Zv3xyW", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-test-registry-*", + "count": 3, + "examples": [ + { + "name": "tv-test-registry-ESqI4b", + "kind": "dir" + }, + { + "name": "tv-test-registry-la3yV1", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-testshard-*", + "count": 192, + "examples": [ + { + "name": "tv-testshard-2026-10-01T22-21-05-004Z-2883021-s1.sh", + "kind": "file" + }, + { + "name": "tv-testshard-2026-10-01T22-04-01-804Z-2873890-s3.sh", + "kind": "file" + } + ] + }, + { + "prefix": "tv-toolchain-npm-*", + "count": 2, + "examples": [ + { + "name": "tv-toolchain-npm-5sTMQk", + "kind": "dir" + }, + { + "name": "tv-toolchain-npm-Fmnb69", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-toolchain-policy-*", + "count": 1, + "examples": [ + { + "name": "tv-toolchain-policy-egmSc5", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-toolchain-policy-mismatch-*", + "count": 1, + "examples": [ + { + "name": "tv-toolchain-policy-mismatch-T9vBXh", + "kind": "dir" + } + ] + } + ] + }, + "meminfo": { + "MemTotal": "8149540 kB", + "MemAvailable": "4670476 kB", + "SwapTotal": "0 kB", + "SwapFree": "0 kB", + "Shmem": "2906032 kB" + }, + "lock": null, + "processes": [ + { + "pid": 1, + "command": "init", + "ownerTokens": [] + }, + { + "pid": 424, + "command": "ukp-fs", + "ownerTokens": [] + }, + { + "pid": 427, + "command": "entrypoint.sh", + "ownerTokens": [] + }, + { + "pid": 428, + "command": "sandbox-api", + "ownerTokens": [] + }, + { + "pid": 429, + "command": "MainThread", + "ownerTokens": [] + }, + { + "pid": 11818, + "command": "sh", + "ownerTokens": [] + }, + { + "pid": 11819, + "command": "python3", + "ownerTokens": [] + }, + { + "pid": 11820, + "command": "sleep", + "ownerTokens": [] + } + ], + "tmpReferences": [], + "sandbox": "tv-testshard-x64-poc-03", + "method": "Read-only os.scandir(/tmp), statvfs, selected /proc/meminfo values, process comm and TV_TEST_SURFACE_OWNER only, cwd/fd paths under /tmp, and the shard owner.json if present. Random six-character directory suffixes grouped for readability. No file contents or environment secrets collected." +} diff --git a/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131-sizes.json b/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131-sizes.json new file mode 100644 index 00000000..dff646f1 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131-sizes.json @@ -0,0 +1,39 @@ +{ + "sandbox": "tv-testshard-x64-poc-131", + "capturedAt": "2026-10-01T22:46:46.362Z", + "command": "du -B1 --max-depth=1 /tmp", + "sizes": { + "/tmp/node-compile-cache": 12963840, + "/tmp/playwright-artifacts-*": 0, + "/tmp/television-source-cli-e2e-*": 1059090432, + "/tmp/television-web-bundle-*": 104255488, + "/tmp/xvfb-run.9dfpKV": 4096, + "/tmp/tv-electron-env-*": 565248, + "/tmp/tv-electron-missing-*": 12288, + "/tmp/tv-dynamic-services-*": 32768, + "/tmp/tv-blaxel-token-*": 12288, + "/tmp/.X11-unix": 0, + "/tmp/tv-provider-reports-*": 49152, + "/tmp/tv-provider-contradiction-*": 12288, + "/tmp/tv-provider-corrupt-*": 12288, + "/tmp/tv-provider-combine-*": 0, + "/tmp/tv-provider-stale-owner-*": 0, + "/tmp/tv-provider-leak-*": 12288, + "/tmp/tv-provider-remote-native-*": 12288, + "/tmp/tv-provider-blaxel-wrapper-*": 12288, + "/tmp/tv-provider-task-infra-*": 12288, + "/tmp/tv-provider-distinct-infra-*": 0, + "/tmp/tv-provider-timing-shards-*": 12288, + "/tmp/tv-provider-normalize-*": 24576, + "/tmp/tv-provider-empty-*": 0, + "/tmp/tv-partial-leases-*": 24576, + "/tmp/tv-interrupted-attempt-*": 86016, + "/tmp/playwright-transform-cache-0": 17170432, + "/tmp/tsx-0": 589824, + "/tmp/tv-posthog-preflight-missing-*": 0, + "/tmp/tv-posthog-preflight-*": 4096, + "/tmp/multipart-uploads": 0, + "/tmp": 1197846528 + }, + "exitCode": 0 +} diff --git a/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131.json b/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131.json new file mode 100644 index 00000000..9f43a4bc --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131.json @@ -0,0 +1,502 @@ +{ + "capturedAt": "2026-10-01T22:44:17.626697+00:00", + "tmp": { + "availableBytes": 2137575424, + "totalBytes": 6676103168, + "categories": [ + { + "prefix": ".X*", + "count": 1, + "examples": [ + { + "name": ".X11-unix", + "kind": "dir" + } + ] + }, + { + "prefix": ".org.chromium.Chromium.*", + "count": 3, + "examples": [ + { + "name": ".org.chromium.Chromium.p98fYu", + "kind": "file" + }, + { + "name": ".org.chromium.Chromium.SPumEs", + "kind": "file" + } + ] + }, + { + "prefix": "MozillaUpdateLock-847956029D21FDC7", + "count": 1, + "examples": [ + { + "name": "MozillaUpdateLock-847956029D21FDC7", + "kind": "file" + } + ] + }, + { + "prefix": "multipart-uploads", + "count": 1, + "examples": [ + { + "name": "multipart-uploads", + "kind": "dir" + } + ] + }, + { + "prefix": "node-compile-cache*", + "count": 1, + "examples": [ + { + "name": "node-compile-cache", + "kind": "dir" + } + ] + }, + { + "prefix": "playwright-*", + "count": 2, + "examples": [ + { + "name": "playwright-artifacts-5Qz2J3", + "kind": "dir" + }, + { + "name": "playwright-transform-cache-0", + "kind": "dir" + } + ] + }, + { + "prefix": "television-source-cli-e2e-*", + "count": 267, + "examples": [ + { + "name": "television-source-cli-e2e-0UvOF0", + "kind": "dir" + }, + { + "name": "television-source-cli-e2e-xz6UWF", + "kind": "dir" + } + ] + }, + { + "prefix": "television-web-bundle-*", + "count": 146, + "examples": [ + { + "name": "television-web-bundle-0U3Bq1", + "kind": "dir" + }, + { + "name": "television-web-bundle-0zVhkg", + "kind": "dir" + } + ] + }, + { + "prefix": "tsx-*", + "count": 1, + "examples": [ + { + "name": "tsx-0", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-blaxel-token-*", + "count": 7, + "examples": [ + { + "name": "tv-blaxel-token-7V3eTY", + "kind": "dir" + }, + { + "name": "tv-blaxel-token-FFrYeT", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-dynamic-services-*", + "count": 8, + "examples": [ + { + "name": "tv-dynamic-services-Cvuced", + "kind": "dir" + }, + { + "name": "tv-dynamic-services-L4ir31", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-electron-env-*", + "count": 33, + "examples": [ + { + "name": "tv-electron-env-0RIXYm", + "kind": "dir" + }, + { + "name": "tv-electron-env-0tSmO7", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-electron-missing-*", + "count": 3, + "examples": [ + { + "name": "tv-electron-missing-OoUqnq", + "kind": "dir" + }, + { + "name": "tv-electron-missing-Y711Eu", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-git-askpass.sh", + "count": 1, + "examples": [ + { + "name": "tv-git-askpass.sh", + "kind": "file" + } + ] + }, + { + "prefix": "tv-interrupted-attempt-*", + "count": 3, + "examples": [ + { + "name": "tv-interrupted-attempt-M63Tzl", + "kind": "dir" + }, + { + "name": "tv-interrupted-attempt-NqzTGZ", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-owner-cleanup-*", + "count": 2, + "examples": [ + { + "name": "tv-owner-cleanup-2026-09-29T18-41-53-559Z-1385612-shard-8.json", + "kind": "file" + }, + { + "name": "tv-owner-cleanup-2026-09-11T19-44-47-497Z-2030667-shard-8.json", + "kind": "file" + } + ] + }, + { + "prefix": "tv-partial-leases-*", + "count": 3, + "examples": [ + { + "name": "tv-partial-leases-5Gl2yt", + "kind": "dir" + }, + { + "name": "tv-partial-leases-TZE8QZ", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-posthog-preflight-*", + "count": 1, + "examples": [ + { + "name": "tv-posthog-preflight-N3e9KL", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-posthog-preflight-missing-*", + "count": 1, + "examples": [ + { + "name": "tv-posthog-preflight-missing-utVaiY", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-blaxel-wrapper-*", + "count": 3, + "examples": [ + { + "name": "tv-provider-blaxel-wrapper-YbDL1t", + "kind": "dir" + }, + { + "name": "tv-provider-blaxel-wrapper-s1eNDm", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-combine-*", + "count": 3, + "examples": [ + { + "name": "tv-provider-combine-KWaBaw", + "kind": "dir" + }, + { + "name": "tv-provider-combine-NMu42s", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-contradiction-*", + "count": 3, + "examples": [ + { + "name": "tv-provider-contradiction-B59JgY", + "kind": "dir" + }, + { + "name": "tv-provider-contradiction-JwJxXQ", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-corrupt-*", + "count": 3, + "examples": [ + { + "name": "tv-provider-corrupt-8J3HdF", + "kind": "dir" + }, + { + "name": "tv-provider-corrupt-FPlCy3", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-distinct-infra-*", + "count": 3, + "examples": [ + { + "name": "tv-provider-distinct-infra-TNDzyX", + "kind": "dir" + }, + { + "name": "tv-provider-distinct-infra-r9YzK1", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-empty-*", + "count": 3, + "examples": [ + { + "name": "tv-provider-empty-SfDSYb", + "kind": "dir" + }, + { + "name": "tv-provider-empty-ar7rau", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-leak-*", + "count": 3, + "examples": [ + { + "name": "tv-provider-leak-0FzFHu", + "kind": "dir" + }, + { + "name": "tv-provider-leak-UF5ev3", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-normalize-*", + "count": 3, + "examples": [ + { + "name": "tv-provider-normalize-KGv4so", + "kind": "dir" + }, + { + "name": "tv-provider-normalize-Odm1iY", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-remote-native-*", + "count": 3, + "examples": [ + { + "name": "tv-provider-remote-native-LhC04g", + "kind": "dir" + }, + { + "name": "tv-provider-remote-native-akc5RU", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-reports-*", + "count": 3, + "examples": [ + { + "name": "tv-provider-reports-0Jfsvl", + "kind": "dir" + }, + { + "name": "tv-provider-reports-12iCrO", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-stale-owner-*", + "count": 3, + "examples": [ + { + "name": "tv-provider-stale-owner-BaHmEj", + "kind": "dir" + }, + { + "name": "tv-provider-stale-owner-OX1kvE", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-task-infra-*", + "count": 3, + "examples": [ + { + "name": "tv-provider-task-infra-4VzJKa", + "kind": "dir" + }, + { + "name": "tv-provider-task-infra-QOrMRp", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-provider-timing-shards-*", + "count": 3, + "examples": [ + { + "name": "tv-provider-timing-shards-GD7UrH", + "kind": "dir" + }, + { + "name": "tv-provider-timing-shards-P7dmtU", + "kind": "dir" + } + ] + }, + { + "prefix": "tv-testshard-*", + "count": 222, + "examples": [ + { + "name": "tv-testshard-2026-10-01T22-04-01-804Z-2873890-s8.sh", + "kind": "file" + }, + { + "name": "tv-testshard-2026-10-01T21-41-24-129Z-2861191-s8.sh", + "kind": "file" + } + ] + }, + { + "prefix": "xvfb-run.*", + "count": 1, + "examples": [ + { + "name": "xvfb-run.9dfpKV", + "kind": "dir" + } + ] + } + ] + }, + "meminfo": { + "MemTotal": "8149540 kB", + "MemAvailable": "3077332 kB", + "SwapTotal": "0 kB", + "SwapFree": "0 kB", + "Shmem": "4433052 kB" + }, + "lock": null, + "processes": [ + { + "pid": 1, + "command": "init", + "ownerTokens": [] + }, + { + "pid": 423, + "command": "ukp-fs", + "ownerTokens": [] + }, + { + "pid": 426, + "command": "entrypoint.sh", + "ownerTokens": [] + }, + { + "pid": 427, + "command": "sandbox-api", + "ownerTokens": [] + }, + { + "pid": 428, + "command": "MainThread", + "ownerTokens": [] + }, + { + "pid": 16872, + "command": "sh", + "ownerTokens": [] + }, + { + "pid": 16873, + "command": "python3", + "ownerTokens": [] + }, + { + "pid": 16874, + "command": "sleep", + "ownerTokens": [] + } + ], + "tmpReferences": [], + "sandbox": "tv-testshard-x64-poc-131", + "method": "Read-only os.scandir(/tmp), statvfs, selected /proc/meminfo values, process comm and TV_TEST_SURFACE_OWNER only, cwd/fd paths under /tmp, and the shard owner.json if present. Random six-character directory suffixes grouped for readability. No file contents or environment secrets collected." +} diff --git a/docs/working/test-harness-resource-cleanup-proposal.md b/docs/working/test-harness-resource-cleanup-proposal.md new file mode 100644 index 00000000..39239af9 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-proposal.md @@ -0,0 +1,154 @@ +# Test-harness resource cleanup + +Proposal for independent review. Owner: rose. Base: `origin/main` at `b5477211`. Branch: `thopter/test-harness-resource-cleanup`. The supervisor arranges independent review of the proposal, specs, proofs, and implementation; Josh reviews every spec delta at PR time. No production behavior changes are proposed. + +## Problem and intended result + +Repeated unbundled application loads can consume several GiB of Chromium response-buffer memory. Persistent Blaxel workers also retain temporary test runtimes across runs, reducing the memory and filesystem capacity available to browsers. The [investigation](chromium-insufficient-resources.md) establishes renderer OOM kills and reproduces `ERR_INSUFFICIENT_RESOURCES`; the exact failing allocation syscall remains unobserved. + +This contribution gives generated test files an owner and a cleanup boundary, runs suitable product acceptances against the built application, and retains compact resource evidence when an acceptance attempt fails. A worker should start its next shard without accumulating the preceding shard's disposable files. Cleanup must preserve active jobs, provider services, deliberate dependency caches, and the evidence still being retrieved by a coordinator. + +PR #9 owns the built-application conversion of **“executable theme reset reloads connected Chromium documents under confirmed state.”** This branch leaves that test body to PR #9. Its existing failure is useful investigation evidence, not an obligation to duplicate that fix here. + +## Evidence available before implementation + +The investigation report and its evidence were copied from `33ad4184`. Additional read-only inspections on 2026-10-01 observed: + +| Observation | `poc-131`, the original failing worker | `poc-03`, the GC experiment's worker | +| --- | ---: | ---: | +| Source-CLI runtime directories | 267 | 17 | +| Source-CLI runtimes, allocated bytes | 1,059,090,432 | 55,930,880 | +| Versioned web-bundle directories | 146 | 27 | +| Versioned web bundles, allocated bytes | 104,255,488 | 22,622,208 | +| Total `/tmp`, allocated bytes | 1,197,846,528 | 156,442,624 | +| Root filesystem available bytes at inventory | 2,137,575,424 | 3,700,719,616 | +| Active shard lock / test owner tokens | None observed | None observed | + +The [poc-131 inventory](test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131.json), [poc-03 inventory](test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03.json), and their [poc-131 sizes](test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131-sizes.json) / [poc-03 sizes](test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03-sizes.json) retain the measurements and collection methods. These are later snapshots, not reconstructions of the failed run. Inventory read filenames, process identities, relevant ownership tokens, and `/tmp` descriptor targets; it did not read fixture credentials or arbitrary process environments into artifacts. Size measurements used `du -B1 --max-depth=1 /tmp`. An earlier `du -x` attempt omitted file storage on this sandbox mount and was discarded. + +There are two proven helper leaks: + +- [`getSourceCLIEntry()`](../../test/helpers/product-server.ts) caches a path in module state after creating `television-source-cli-e2e-*`. Its cache is assigned before the build succeeds. `disposeAllProductServers()` removes per-launch homes and bundled copies, but has no cleanup for this shared runtime. +- [`buildVersionedWebBundle()`](../../test/helpers/versioned-web-bundle.ts) caches one promise per version and creates `television-web-bundle-*`, with no removal on worker completion or build failure. This is additional evidence supporting run-owned scratch space. Its worker cleanup belongs in the same contribution: it is another concrete cached test build, not a hypothetical extension. + +Other `/tmp` entries include Playwright artifacts and transform caches, Node's compile cache, `tsx` caches, Chromium temporary files, X11 state, an `xvfb-run` authority directory, the Git askpass helper, hundreds of uploaded shard scripts, test-runner fixtures, and `multipart-uploads`. The idle process tables include Blaxel's `sandbox-api`, `ukp-fs`, and entrypoint processes. The precise owner of every miscellaneous entry was not established; in particular, the upload directory must be treated as outside Television's disposal ownership. Neither idle snapshot had an open `/tmp` descriptor or cwd reference, which establishes only those instants. + +## Can we clean `/tmp` safely? + +**Yes, for temporary files whose ownership and lifetime the runner establishes. A blanket deletion of `/tmp` is unsafe.** The worker lease itself lives there, as do provider and display infrastructure. Name prefixes or age alone do not establish that a file is unused. + +### Concurrency and the present lease boundary + +The canonical coordinator intends one shard job per sandbox. [`acquireAny()` and `lockCommand()`](../../scripts/run-blaxel-testshards.mjs) acquire `/tmp/tv-testshard.lock`; another coordinator skips an unexpired lock. A normal broad run uses separate workers for its shards. Within a shard, surface tasks execute sequentially, but a surface can have several test workers and many child processes. For example, the browser-app surface uses two Playwright workers on CI; browser-app-real-stack uses one. + +This is not evidence that a sandbox can never have overlapping jobs: + +1. Lock expiry currently allows reclamation based on time alone. The lock records the coordinator's host/PID, not the remote shard supervisor's live identity. Setup processes are not all covered by surface owner tokens. The stale-owner reaper deliberately leaves live-supervisor tokens alone and can exit successfully with `liveOwners` present. +2. The shard shell's `EXIT` trap releases its lock before `runShard()` downloads results from the fixed `/workspace/television/.testshards/results` directory. A following job can reset those files while the prior coordinator reads them. +3. Direct Blaxel process API calls are not serialized by this repository lock. The read-only inventory itself used that API. The inspection found no overlapping test jobs, but does not establish a history-wide absence of overlap. + +The proposal therefore extends the existing lease boundary as part of cleanup. The coordinator retains its lease through result retrieval and disposal, and the worker does not independently unlock at shell exit. Lease reclamation and remote dispatch must confirm ownership and the absence of a live prior shard supervisor using process start identity, including setup. An expired timestamp alone cannot authorize deleting that job's files or starting a second checkout. Acquisition/reclamation/release must preserve exclusive ownership under racing coordinators. An ambiguous owner or failed process inspection keeps the worker unavailable and reports infrastructure failure. Existing stale-process attribution and TERM/KILL cleanup remain the mechanism for abandoned descendants. + +This is a narrowly required correction to worker ownership, not a redesign of shard scheduling or a change to retry budgets. No observed Chromium failure has been attributed to these lease races. + +### A directory for each shard attempt + +Give every leased shard **attempt** a fresh, short, private directory beneath a runner-owned root, for example `/tmp/tv-testshards//`. Record its run, shard, attempt, lease identity, and remote supervisor PID/start identity outside its disposable `tmp/` child. Use a short path because browser and Electron processes can create Unix sockets with limited pathname length. + +Export an absolute `TMPDIR` pointing to that child before running preparation and test commands. Keep scripts, plans, askpass, and setup sidecars within the attempt directory as well. Preserve `TMPDIR` through child environments. Continue to keep deliberately persistent npm, Electron, browser-download, and dependency-hash caches at their existing locations outside scratch space. Playwright's transform cache may become attempt-local initially; measure any resulting setup cost before deciding it deserves a separately bounded persistent cache. + +The mechanism is supported by both source and installed dependencies: + +- On non-Windows systems, [Node's `os.tmpdir()`](https://nodejs.org/api/os.html#ostmpdir) checks `TMPDIR` before `TMP`, `TEMP`, and `/tmp`. Both cached build helpers use `os.tmpdir()`. +- [Chromium 148's `GetShmemTempDir()` and `GetTempDir()`](https://github.com/chromium/chromium/blob/148.0.7778.96/base/files/file_util_posix.cc#L1283-L1302) select the temporary directory when `--disable-dev-shm-usage` is present; `GetTempDir()` reads `TMPDIR`. The investigation observed this flag in the actual browser launch. +- Installed Playwright 1.58.2 uses `os.tmpdir()` for browser profiles and artifact directories in `playwright-core/lib/server/browserType.js`, inherits the parent environment by default, and includes that Chromium flag. Its transform cache also defaults to `os.tmpdir()`. + +This establishes the intended wiring; a live Blaxel test must still observe Node-created paths and actual Chromium shared-memory descriptor paths under the attempt directory. We have not run that check at the proposal stage. Setting `TMPDIR` cannot relocate hard-coded `/tmp` paths or X11's global socket directory; the runner's explicit temporary paths must be moved deliberately, and external infrastructure remains untouched. Chromium also unlinks shared-memory files while holding them open, so directory deletion cannot substitute for stopping the owning processes. + +Cleanup follows three ownership boundaries: + +1. **Before an attempt:** after exclusive ownership and the process audit are established, reclaim earlier runner-owned attempt directories whose recorded supervisors and descendants are gone. Revalidate ownership before removal, reject unexpected directory/symlink layouts, and stay inside the dedicated namespace. Keep live or uncertain owners untouched. Create this attempt's directory fresh instead of deleting a guessed path in advance. +2. **After execution:** finalize resource evidence outside disposable scratch, complete process teardown, then remove that attempt's scratch. Failure, timeout, and interruption use the same ordering. The coordinator's existing unconfirmed-execution path must confirm process cleanup before it removes scratch or releases the lease. +3. **After retrieval:** remove retained worker reports only after download completes, then release the matching lease. Keep artifact paths specific to the attempt so a lost lease cannot make a coordinator read a different job's report. A killed coordinator can leave a directory; the next safe lease reclaims it through the same ownership check. It does not need a separate cleanup daemon. + +Runner reports should use an attempt-specific directory that survives checkout reset. The coordinator already validates artifact roots; extend that check to the exact root it allocated, rather than accepting arbitrary paths advertised by a worker. This removes the fixed-results-directory race while retaining current normalized report identities. + +### Files that predate ownership metadata + +Existing `television-source-cli-e2e-*` and `television-web-bundle-*` directories lack a reliable recorded owner. Routine new-run cleanup will not infer one from a prefix. For the initial rollout, use a **drained worker under an exclusive maintenance lease**, with no executing shard or other consumer, to inventory and remove only these verified generated runtime trees. Check directory type, owner, and expected runtime/build layout; preserve unrelated `/tmp` entries and deliberate caches. A worker that cannot be established idle is skipped. Where a worker's contents or active owners cannot be established, drain and reprovision it through the existing pool procedure instead of widening deletion. + +This one-time operation needs coordination with whoever is dispatching runs, since older coordinators still use the present lease protocol. It should be performed after the new protocol has converged and old jobs have drained. This proposal authorizes no immediate deletion on the shared pool; none has been done. The normal before/after cleanup covers future runs regardless of which helper created their temporary files. + +## Worker lifetime for cached runtimes + +Keep one lazily built source-CLI runtime per test worker and reuse it across launches and test cases. Publish the cached entry only after the build and asset installation succeed. A partial build removes its own directory and leaves the cache retryable. Apply the equivalent ownership rule to the cached versioned web bundles, retaining their per-version reuse. + +Test cleanup continues to stop each case's product servers promptly. Worker teardown then stops any remaining servers and removes the shared cached runtime and versioned bundles. Cleanup is idempotent and waits for pending builds before removing their outputs. The worker teardown must cover both the shared Playwright registration helper and the Vitest callers; a per-test deletion would discard reuse and add unnecessary builds. A failed server teardown cannot justify removing files beneath a surviving server. The outer shard's process ownership and scratch cleanup provide the backstop when a worker is killed and cannot execute its own teardown. + +The relevant entry points are `test/helpers/playwright.ts`, the two cached build helpers, and the Node/Vitest callers of those helpers. The design should use the runners' existing teardown facilities rather than a general-purpose cleanup registry. Failure during a per-launch runtime copy must also dispose that copy and its newly created home: the current bundle preparation occurs before the launch disposer is registered. + +## Built application for product acceptances + +Use the already-built web application served by `ProductServer`. Existing registry pre-commands build the web and artifact documents before both browser-app surfaces, and `installProductRuntimeAssets()` already copies the application index and assets. Preserve the tests' assertions, real browser behavior, CSS-motion declarations, authentication, and server lifecycle. + +The proposed conversion set is the browser product walks using `ProductServer.appURL()`: + +| Tests under `packages/web/test/e2e/` | Reason and boundary | +| --- | --- | +| `channel-product.test.ts`, `channel-crud.test.ts`, `channel-sidebar-remote.test.ts`, `channel-switcher-context-menu.test.ts` | Real channel actions, shared state, drag behavior, and document continuity; no claim about Vite modules. | +| `tab-pages.test.ts`, `keyboard-navigation-product.test.ts`, `artifact-view-wheel.test.ts` | Browser product behavior and real artifact frames. | +| Product-server walks in `top-bar.test.ts` and `auth.01.test.ts` | Built-shell selection, creation, and token behavior. Keep their component fixtures and separate endpoint tests in their current harnesses. | +| `browser-demo-mode.test.ts` | Real server marker and artifact navigation; built loading preserves the behavior being asserted. | +| `markdown-content-sync.spec.ts`, `path-artifact-saga.test.ts`, `path-artifact-real-stack-coverage.test.ts`, `navigable-history-real-stack.01.test.ts`, `navigable-history-real-stack.02.test.ts` | File watching, persistence, editing, and navigation through real product boundaries. A test title saying “source-served” is not itself a requirement for Vite; reconcile its proof citation. | +| `onboarding-browser.test.ts` | Real boot/restart and browser-visible onboarding. Preserve the stable browser origin while the backend binds a new port. | +| The settings-persistence and executable-theme-lifecycle cases in `theme-product-acceptance.test.ts` | Both exercise real server restart, and the executable lifecycle loads two pages repeatedly. Preserve those transitions through a stable origin serving built assets. Leave the theme-reset case to PR #9. | + +A server URL is sufficient where a backend stays up. Where restart is part of the walk, reuse the existing [`startStableFrontProxy()`](../../test/helpers/stable-front-proxy.ts) to serve the built document and retarget HTTP/WebSocket traffic after restart. Do not weaken a reconnect test by navigating to a replacement origin or clearing browser state. Keep development loading available explicitly for module and fixture seams; avoid changing every `appURL()` caller implicitly, including PR #9's test and the Electron dispatcher seams. + +The product runtime also needs the built `views/url-unsupported` assets for the existing external-navigation walk, which currently obtains that view from Vite. Add the missing test-runtime asset copy, not a production routing change. This is a prerequisite for preserving that assertion honestly. + +Other suites that construct an in-process `Server`, component fixtures, and Electron-specific module/dispatcher seams are outside this initial conversion set. They are not asserted to require Vite forever. Converting this concrete set addresses the identified product walks without refactoring every browser harness. Existing built-app acceptances remain built-app acceptances. + +## Failure diagnostics and retention + +Collect small resource observations in the parent surface process so that a browser or test-worker crash cannot destroy all the evidence. Cover the canonical planned shard and target paths, including their preparation and teardown failures. The shared browser helper supplements those observations with attempt identity and browser failures. The parent data remains useful if the attempt helper never finishes. + +Retain the following for failed attempts, including attempts that later recover on retry: + +- Timestamped available memory and shared memory, available bytes/inodes for the effective temporary filesystem and `/dev/shm`, and initial/peak/final observations from a modest periodic sampler. +- The Linux `oom_kill` counter before and after execution; available cgroup memory-limit/OOM counters; a bounded kernel OOM excerpt when readable. Unavailable probes are explicit, not fabricated zeroes. A system counter increase alone is not attribution to a particular browser. +- Browser version, expected Playwright browser revision, browser crash/disconnect and failed-request error names, and the failed attempt's file/title/project/retry identity. Omit credentials, request query strings, response bodies, and arbitrary process environments. +- On failure, owned process identities and memory/descriptor summaries sufficient to compare with a descriptor limit and correlate kernel PID evidence. Avoid continuous full descriptor scans or permanent NetLog capture. + +Prefer one resource sidecar per surface plus small attempt records over a separate monitoring service. Keep it incrementally readable, with explicit truncation/drop counts, and expose retained paths through the normal task/provider reporting. Download the sidecars through the same checked artifact path as native reports. Probe failure produces an observability warning and preserves the original test result. It must neither hide a failure nor create a passing result from missing test evidence. + +The proposed bound is **1 MiB per surface resource sidecar and 8 MiB total resource diagnostics per shard attempt**, retaining initial state, extrema, OOM deltas, and recent failure context when the history is truncated. This cap applies while recording, not only at teardown. Do not add default videos, heap snapshots, or full network logs. Existing trace policy remains unchanged. + +On a completed download, the coordinator retains the evidence in its canonical run directory and removes the worker copy. If retrieval is interrupted, bounded diagnostics remain with the stopped attempt until the next safe lease; that lease may replace them after recording that leftover evidence was reclaimed. Ambiguous/live owners prevent reclamation and new dispatch. Thus idle persistent workers do not accumulate an unbounded sequence of diagnostic runs. Coordinator-side `.test-runs/` retention is unchanged. + +## Authority and derivation + +After proposal convergence, derive the smallest changes in the existing owners: + +- [`testing-policy.md`](../../specs/arch/testing-policy.md): built application as the default for product browser acceptance when development-module behavior is not under test; worker-owned reusable test runtimes and cleanup expectations. +- [`test-runner.md`](../../specs/arch/test-runner/test-runner.md): resource observation attached to the surface lifecycle, with process teardown preceding disposal of owned temporary files. +- [`blaxel-testshards.md`](../../specs/arch/test-runner/blaxel-testshards.md): attempt temporary directories, exclusive leases across setup/execution/retrieval/cleanup, safe orphan reclamation, and persistent-worker retention. +- [`sharded-execution.md`](../../specs/arch/test-runner/sharded-execution.md) and [`reporting.md`](../../specs/arch/test-runner/reporting.md): resource artifact references, bounded content, unknown/unavailable evidence, and preservation across attempts and downloads. Touch [`preflight.md`](../../specs/arch/test-runner/preflight.md) only where ownership checks actually change its stated contract. + +Update the mirrored proofs after independent spec convergence. Test setup changes may also require proof-only corrections to descriptions of the affected browser walks; they do not change their product promises. Do not prescribe a new framework or adopt unrelated harness internals into spec authority. + +## Validation and contribution boundaries + +All execution is on Blaxel at committed, pushed revisions. The shared development host performs file inspection, edits, Git operations, and lightweight remote coordination only. No tests, builds, browser launches, lint, or type checks have run for this proposal. Apply the test-iteration discipline: red/green at the narrowest relevant file, then affected surfaces, then full verification with the default retry budget. + +The proofs should establish a small number of consequential boundaries: reuse and final disposal of real runtime files; partial-build cleanup; preservation of a live job's files while orphaned owned scratch is reclaimed; real environment propagation into Node and Chromium; the result-download/lease boundary; and retained, bounded failure evidence when an attempt fails or its worker exits. Contract coverage handles unavailable resource probes and retention limits. A live targeted Blaxel run and a live planned shard run establish the provider paths. Exercise interruption with disposable owned subprocesses on an isolated Blaxel worker, without intentionally exhausting memory or running repository fixtures restricted to isolated GitHub hosts. + +Re-run the converted acceptances without weakening their user-visible assertions. A recorded built-page request inventory should show that they no longer load the unbundled application graph. Validate restart continuity and authenticated navigation directly. Repeated use of one isolated worker should leave no generated runtime directories after cleanup and no growing sequence of resource reports. + +`npm run verify -- blaxel` still runs lint/type-check/package-manifest phases on the invoking machine. Run that command from a dedicated Blaxel coordinator host for the final gate; invoking it here would violate the host invariant. Record the exact tested revision, failures, recovered flakes, and any measurement limits. Ready-PR CI remains a merge requirement of the repository, not a substitute for the required Blaxel evidence. + +Keep browser-version alignment, shard count, browser request limits, retries, and production memory behavior outside this contribution. Forced GC is an investigation tool, not a proposed mitigation. The demo-mode and drag flakes have not been proven to share the resource-exhaustion cause. + +## Review status + +Proposal: awaiting independent review. No spec, proof, test, or implementation edits yet. The supervisor will receive the pushed proposal commit and draft PR. Subsequent work waits for that review; Josh's spec acceptance remains due at PR time. From 99b7ed93f396cc7c14b54c281bfcfa08e2d7abe3 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:15:08 +0000 Subject: [PATCH 02/64] Narrow resource cleanup proposal after independent review --- docs/working/blaxel-results-download-race.md | 21 +++ .../test-harness-resource-cleanup-proposal.md | 93 +++++++++---- ...test-harness-resource-cleanup-review-r1.md | 131 ++++++++++++++++++ ...arness-resource-cleanup-review-response.md | 28 ++++ 4 files changed, 243 insertions(+), 30 deletions(-) create mode 100644 docs/working/blaxel-results-download-race.md create mode 100644 docs/working/test-harness-resource-cleanup-review-r1.md create mode 100644 docs/working/test-harness-resource-cleanup-review-response.md diff --git a/docs/working/blaxel-results-download-race.md b/docs/working/blaxel-results-download-race.md new file mode 100644 index 00000000..ba367eb7 --- /dev/null +++ b/docs/working/blaxel-results-download-race.md @@ -0,0 +1,21 @@ +# Blaxel result retrieval can race the next checkout + +Follow-up identified while investigating Chromium resource failures. This item is outside draft PR #12's cleanup implementation. No observed Chromium failure has been attributed to this race. No change to the lease protocol is authorized by this document. + +## Evidence and consequence + +At `b5477211`, `scripts/run-blaxel-testshards.mjs` does the following: + +1. The remote shard script's `EXIT` trap releases `/tmp/tv-testshard.lock` at shell exit. +2. `runShard()` waits for that remote process to finish, then downloads the summary and artifacts from `/workspace/television/.testshards/results`. +3. A new lease can start another checkout, whose preparation removes `.testshards`, before the prior coordinator finishes downloading. + +The source establishes an unprotected interval, not an observed collision. A collision could lose the previous job's evidence or make it encounter another job's files. Existing report identity validation remains necessary but cannot recover deleted evidence. + +## Scope for a separate proposal + +Assess retaining the existing lease through retrieval versus using immutable attempt-specific report locations, with bounded orphan retention in either case. Work through coordinator interruption, download failure, worker reuse, and compatibility with older branches that still unlock at shell exit and reset the fixed results root. Compare the smallest correction before adding lease machinery. + +TTL recovery is a separate availability decision. The current lock lives 900/1200 seconds against a worker timeout of 240/480 seconds; the Blaxel spec deliberately leaves uncertain cleanup to TTL as a final bound. Whether the provider can leave a supervisor alive beyond that timeout remains unproven. Replacing TTL with indefinite quarantine would require Josh to choose its availability cost and an explicit operator recovery procedure. Scratch ownership checks do not require that policy change. + +The cleanup proposal preserves the existing results location and lease behavior. Its diagnostics inherit this download-race limitation. This follow-up has no assigned implementation or rollout date. diff --git a/docs/working/test-harness-resource-cleanup-proposal.md b/docs/working/test-harness-resource-cleanup-proposal.md index 39239af9..8cc32ff3 100644 --- a/docs/working/test-harness-resource-cleanup-proposal.md +++ b/docs/working/test-harness-resource-cleanup-proposal.md @@ -6,7 +6,9 @@ Proposal for independent review. Owner: rose. Base: `origin/main` at `b5477211`. Repeated unbundled application loads can consume several GiB of Chromium response-buffer memory. Persistent Blaxel workers also retain temporary test runtimes across runs, reducing the memory and filesystem capacity available to browsers. The [investigation](chromium-insufficient-resources.md) establishes renderer OOM kills and reproduces `ERR_INSUFFICIENT_RESOURCES`; the exact failing allocation syscall remains unobserved. -This contribution gives generated test files an owner and a cleanup boundary, runs suitable product acceptances against the built application, and retains compact resource evidence when an acceptance attempt fails. A worker should start its next shard without accumulating the preceding shard's disposable files. Cleanup must preserve active jobs, provider services, deliberate dependency caches, and the evidence still being retrieved by a coordinator. +The fixes have different effects. The theme-reset walk added roughly 3.2 GB of shared memory in a passing run that reached about 0.55 GB available RAM. Switching that walk to the built application removes its hundreds of development-module requests per load; the expected memory saving is large, but still needs a post-change measurement. By comparison, the two leaked runtime prefixes occupied 1.16 GB on the worst worker inspected and 0.079 GB on the other. All of `/tmp` occupied 1.20 GB and 0.16 GB respectively. Cleanup recovers accumulated capacity; it does not make repeated unbundled loading cheap or guarantee enough headroom. The original worker also retained roughly 1.4 GiB of checkout/dependencies, 0.9 GiB of browser installations, and other intentional caches. Those are outside this cleanup. + +This contribution fixes the cached runtime lifetimes, runs suitable product acceptances against the built application, and retains compact resource evidence when an acceptance fails. For runs using the updated harness, owned disposable files should be removed after confirmed process teardown, with orphan cleanup on a later run when teardown could not finish. Older branches will continue to leave unowned files until updated. Cleanup must preserve active jobs, provider services, deliberate dependency caches, and reports outside scratch space. PR #9 owns the built-application conversion of **“executable theme reset reloads connected Chromium documents under confirmed state.”** This branch leaves that test body to PR #9. Its existing failure is useful investigation evidence, not an obligation to duplicate that fix here. @@ -35,7 +37,7 @@ Other `/tmp` entries include Playwright artifacts and transform caches, Node's c ## Can we clean `/tmp` safely? -**Yes, for temporary files whose ownership and lifetime the runner establishes. A blanket deletion of `/tmp` is unsafe.** The worker lease itself lives there, as do provider and display infrastructure. Name prefixes or age alone do not establish that a file is unused. +**After a run:** yes, remove that run's own scratch after its processes stop and before the existing exit path releases the worker. **Before a run:** yes, reclaim owned scratch only when its recorded supervisor and descendants are confirmed gone. **Existing unowned files:** require a separate rollout decision from Josh, described below. Never delete all of `/tmp`: the worker lease itself lives there, as do provider and display infrastructure. Name prefixes or age alone do not establish that a file is unused. ### Concurrency and the present lease boundary @@ -43,17 +45,17 @@ The canonical coordinator intends one shard job per sandbox. [`acquireAny()` and This is not evidence that a sandbox can never have overlapping jobs: -1. Lock expiry currently allows reclamation based on time alone. The lock records the coordinator's host/PID, not the remote shard supervisor's live identity. Setup processes are not all covered by surface owner tokens. The stale-owner reaper deliberately leaves live-supervisor tokens alone and can exit successfully with `liveOwners` present. +1. Lock expiry currently allows reclamation based on time alone. The lock records the coordinator's host/PID, not the remote shard supervisor's live identity. Setup processes are not all covered by surface owner tokens. The stale-owner reaper deliberately leaves live-supervisor tokens alone and can exit successfully with `liveOwners` present. However, the configured TTL is 900/1200 seconds against a 240/480-second worker timeout. Expiry is deliberately the spec's final availability bound when cleanup cannot be confirmed. Whether the provider can leave a job alive past that bound was not established. 2. The shard shell's `EXIT` trap releases its lock before `runShard()` downloads results from the fixed `/workspace/television/.testshards/results` directory. A following job can reset those files while the prior coordinator reads them. 3. Direct Blaxel process API calls are not serialized by this repository lock. The read-only inventory itself used that API. The inspection found no overlapping test jobs, but does not establish a history-wide absence of overlap. -The proposal therefore extends the existing lease boundary as part of cleanup. The coordinator retains its lease through result retrieval and disposal, and the worker does not independently unlock at shell exit. Lease reclamation and remote dispatch must confirm ownership and the absence of a live prior shard supervisor using process start identity, including setup. An expired timestamp alone cannot authorize deleting that job's files or starting a second checkout. Acquisition/reclamation/release must preserve exclusive ownership under racing coordinators. An ambiguous owner or failed process inspection keeps the worker unavailable and reports infrastructure failure. Existing stale-process attribution and TERM/KILL cleanup remain the mechanism for abandoned descendants. +**The existing lease protocol, TTL, and result location remain unchanged in this contribution.** Safe scratch cleanup uses a fresh private directory and that directory's process ownership, independently of whether the lease has changed hands. A live or uncertain owner prevents removal of that directory; it does not add a permanent worker quarantine. Existing stale-process attribution and TERM/KILL cleanup remain in force. Cleanup cannot make overlapping checkouts safe and does not claim to do so. -This is a narrowly required correction to worker ownership, not a redesign of shard scheduling or a change to retry budgets. No observed Chromium failure has been attributed to these lease races. +The result-download race is recorded as a [separate follow-up](blaxel-results-download-race.md). Resolving it, or replacing TTL recovery with operator intervention, needs its own scope and consequences assessed, including compatibility with older coordinators. Neither is required for this scratch cleanup. No observed Chromium failure has been attributed to these lease weaknesses. ### A directory for each shard attempt -Give every leased shard **attempt** a fresh, short, private directory beneath a runner-owned root, for example `/tmp/tv-testshards//`. Record its run, shard, attempt, lease identity, and remote supervisor PID/start identity outside its disposable `tmp/` child. Use a short path because browser and Electron processes can create Unix sockets with limited pathname length. +Give every leased shard **attempt** a fresh, short, private directory beneath a runner-owned root, for example `/tmp/tv-testshards//`. Record its run, shard, attempt, lease identity, and remote supervisor PID/start identity outside its disposable `tmp/` child. Record ownership before using scratch; the pre-dispatch directory must not be mistaken for a dead supervisor. Upload/dispatch failure removes only the directory this coordinator created when execution is known not to have started. An unknown dispatch outcome leaves it for confirmed recovery. Use a short path because browser and Electron processes can create Unix sockets with limited pathname length. Export an absolute `TMPDIR` pointing to that child before running preparation and test commands. Keep scripts, plans, askpass, and setup sidecars within the attempt directory as well. Preserve `TMPDIR` through child environments. Continue to keep deliberately persistent npm, Electron, browser-download, and dependency-hash caches at their existing locations outside scratch space. Playwright's transform cache may become attempt-local initially; measure any resulting setup cost before deciding it deserves a separately bounded persistent cache. @@ -65,19 +67,28 @@ The mechanism is supported by both source and installed dependencies: This establishes the intended wiring; a live Blaxel test must still observe Node-created paths and actual Chromium shared-memory descriptor paths under the attempt directory. We have not run that check at the proposal stage. Setting `TMPDIR` cannot relocate hard-coded `/tmp` paths or X11's global socket directory; the runner's explicit temporary paths must be moved deliberately, and external infrastructure remains untouched. Chromium also unlinks shared-memory files while holding them open, so directory deletion cannot substitute for stopping the owning processes. -Cleanup follows three ownership boundaries: +Cleanup has two paths: + +1. **Before an attempt:** reclaim earlier directories in the dedicated namespace only when recorded PID/start identities are dead and no attributed descendants remain. Revalidate before removal and reject unexpected directory/symlink layouts. Missing metadata or unreadable process evidence means skip and report, not guess. A live supervisor protects its directory even if its lease has expired. Create the new attempt's directory fresh; never delete a guessed path to make room. +2. **After execution:** finalize evidence in the existing results directory outside scratch, complete process teardown, then remove this attempt's scratch from the existing exit path before it releases the lock. Setup failure, test failure, timeout, and interruption follow the same ownership rule. The coordinator's existing unconfirmed-execution path removes scratch only after its process cleanup is confirmed. Otherwise leave it for a later ownership-checked reclamation. Preserve the original outcome if cleanup itself fails, and report the leftover path. + +The supervisor identity alone is not enough if detached descendants survived it. Reuse the existing owner-token audit and attach the attempt identity to setup children as needed for attribution; do not kill unrelated processes to make a directory reclaimable. A missing/uncertain owner can leave residual scratch requiring operator attention, but does not change lease availability. This is a disposal safety rule, not a promise that every abandoned byte can always be reclaimed automatically. + +Reports stay under the current checked results root and are downloaded normally. Scratch cleanup does not remove them. This retains the known download race as an explicit limit instead of redesigning report storage inside this contribution. -1. **Before an attempt:** after exclusive ownership and the process audit are established, reclaim earlier runner-owned attempt directories whose recorded supervisors and descendants are gone. Revalidate ownership before removal, reject unexpected directory/symlink layouts, and stay inside the dedicated namespace. Keep live or uncertain owners untouched. Create this attempt's directory fresh instead of deleting a guessed path in advance. -2. **After execution:** finalize resource evidence outside disposable scratch, complete process teardown, then remove that attempt's scratch. Failure, timeout, and interruption use the same ordering. The coordinator's existing unconfirmed-execution path must confirm process cleanup before it removes scratch or releases the lease. -3. **After retrieval:** remove retained worker reports only after download completes, then release the matching lease. Keep artifact paths specific to the attempt so a lost lease cannot make a coordinator read a different job's report. A killed coordinator can leave a directory; the next safe lease reclaims it through the same ownership check. It does not need a separate cleanup daemon. +### Mixed versions and existing unowned files: Josh's rollout decision -Runner reports should use an attempt-specific directory that survives checkout reset. The coordinator already validates artifact roots; extend that check to the exact root it allocated, rather than accepting arbitrary paths advertised by a worker. This removes the fixed-results-directory race while retaining current normalized report identities. +Every coordinator uploads the script from its own branch. Older branches keep creating `television-source-cli-e2e-*` and `television-web-bundle-*` directly under `/tmp`, bypass the new attempt directory, and will keep leaking them. One cleanup now cannot end that accumulation. The updated harness cleans files created under its own scratch, including helpers not individually repaired; it cannot make that claim for old-branch runs or hard-coded external paths. -### Files that predate ownership metadata +**Recommendation for Josh's decision:** accept temporary residue while active branches take the change, then arrange a drained-pool recreation using the existing delete-then-ensure procedure. This needs a coordinated pause in dispatch and pays cold provisioning/dependency costs, but removes all old test residue with existing machinery. If capacity becomes a problem sooner, Josh can schedule an earlier recreation knowing that old branches can contaminate the fresh workers again. No date or shared-pool deletion is authorized by this proposal. -Existing `television-source-cli-e2e-*` and `television-web-bundle-*` directories lack a reliable recorded owner. Routine new-run cleanup will not infer one from a prefix. For the initial rollout, use a **drained worker under an exclusive maintenance lease**, with no executing shard or other consumer, to inventory and remove only these verified generated runtime trees. Check directory type, owner, and expected runtime/build layout; preserve unrelated `/tmp` entries and deliberate caches. A worker that cannot be established idle is skipped. Where a worker's contents or active owners cannot be established, drain and reprovision it through the existing pool procedure instead of widening deletion. +| Option | Benefit | Cost and consequence | +| --- | --- | --- | +| Accept residue during migration, then drain and recreate the pool (recommended) | No new legacy-deletion code; clears every worker and every old disposable prefix. | Temporary reduced headroom, a coordinated interruption, and cold caches. May need repetition if old branches continue running. | +| Operator removes the two known prefixes from explicitly drained workers | Preserves dependency and browser caches. | Must establish that no job or other consumer uses those files; only recovers these two prefixes; needs repetition during mixed versions. This would be an operator procedure, not routine prefix-based deletion in this PR. | +| Add routine legacy-prefix cleanup | Reclaims old-branch leaks repeatedly. | Adds deletion without recorded ownership, dependent on proving the whole worker idle despite mixed coordinators; disproportionate to this fix and not recommended. | -This one-time operation needs coordination with whoever is dispatching runs, since older coordinators still use the present lease protocol. It should be performed after the new protocol has converged and old jobs have drained. This proposal authorizes no immediate deletion on the shared pool; none has been done. The normal before/after cleanup covers future runs regardless of which helper created their temporary files. +Josh must choose the shared-pool rollout option and timing. The contribution can be reviewed and implemented without doing that operation; it cannot claim the whole pool is clean until the decision is executed and old branches stop leaking. No shared-pool or development-host cleanup has been performed. ## Worker lifetime for cached runtimes @@ -87,6 +98,8 @@ Test cleanup continues to stop each case's product servers promptly. Worker tear The relevant entry points are `test/helpers/playwright.ts`, the two cached build helpers, and the Node/Vitest callers of those helpers. The design should use the runners' existing teardown facilities rather than a general-purpose cleanup registry. Failure during a per-launch runtime copy must also dispose that copy and its newly created home: the current bundle preparation occurs before the launch disposer is registered. +The helper fix matters even with shard scratch: it releases files between test workers within a shard and protects callers on hosts without the Blaxel shard script, including ordinary CI and developer environments. The reviewer independently counted 1,388 source-CLI and 727 versioned-bundle directories on this development host; that is reviewer evidence, not a new inspection by this contributor. Cleaning those existing host files is the host owner's decision and outside this contribution. The helper fix prevents new leakage on updated branches; it does not retrospectively remove unowned files. + ## Built application for product acceptances Use the already-built web application served by `ProductServer`. Existing registry pre-commands build the web and artifact documents before both browser-app surfaces, and `installProductRuntimeAssets()` already copies the application index and assets. Preserve the tests' assertions, real browser behavior, CSS-motion declarations, authentication, and server lifecycle. @@ -109,31 +122,47 @@ The product runtime also needs the built `views/url-unsupported` assets for the Other suites that construct an in-process `Server`, component fixtures, and Electron-specific module/dispatcher seams are outside this initial conversion set. They are not asserted to require Vite forever. Converting this concrete set addresses the identified product walks without refactoring every browser harness. Existing built-app acceptances remain built-app acceptances. -## Failure diagnostics and retention +### Remaining development-page exposure + +At `b5477211`, a read-only inventory finds 38 e2e test files with a literal `/packages/web/src/index.html` reference and two more using `guerilla-artifact-sharing.helpers.ts`: 40 test files, or 41 source-loading files when that helper is counted. Repository tests that inspect HTML as text are excluded. The proposed 17-file conversion set plus PR #9 leaves 23 of those test files, including the three desktop `appURL()` users. This count is a source inventory, not a measurement of concurrent pages or memory. Component fixtures importing individual modules are a separate exposure and are not counted as full application loads. + +| Remaining tests | Observed reload/page behavior and why deferred | +| --- | --- | +| Web `telemetry-session.test.ts` | One case reloads the application; another opens two tabs in one persistent context, closes it, then opens a new context. This is a concrete remaining multi-page exposure. It uses an in-process server with an injected telemetry sink; converting that boundary needs its own proof review. | +| Web `artifact-navigation.01.test.ts`, `artifact-navigation-history.test.ts` | Their `beforeEach` loads the development app, clears storage, then reloads it: two full graph loads per case. They use directly controlled `Store`/`Server` instances. These are sensible next conversion candidates, not proven-safe exceptions. | +| Web `auth.02.test.ts` | The silent reconnect case explicitly reloads the app. Other cases inject authentication failures into an in-process server. PR #9 also changes this file, so reassess its final walks after that lands. | +| Web `reload-after-reconnect.01/.02.test.ts`, `guerilla-artifact-sharing.test.ts`, `guerilla-artifact-sharing-resilience.test.ts` | These open one application page and reload/recover artifact documents or reconnect the event socket. The reconnect continuity case explicitly expects no top-level navigation. They control server internals/producer-consumer seams; an artifact reload is not evidence of a full application graph reload. | +| Web `settings.test.ts`, `platform-marker.test.ts`, `artifact-navigation.02.test.ts`, `appearance-delivery.test.ts`, `theme.test.ts`, `artifact-full-screen.test.ts` | The inspected source-app cases use one application page. Other cases in these files use fixtures or built assets. Retain their in-process server, platform, appearance, and navigation boundaries for this contribution; no claim that development modules are inherently necessary. | +| Desktop `all-webview-dispatcher.01/.02.test.ts`, `webview-event-parity.test.ts` | The executable-theme parity walk explicitly waits for a top-level document reset and later closes/relaunches Electron. The navigation-history walk quits and relaunches. These remain exposed; application resets, fresh Electron processes, and guest-webview reloads must be distinguished in any follow-up conversion. | +| Desktop `artifact-view-webview-wheel.test.ts`, `guerilla-artifact-sharing.test.ts`, `path-webview-content-reload.test.ts`, `artifact-navigation.test.ts`, `native-navigation-key.test.ts`, `artifact-navigation-external.test.ts` | Launch the development application in Electron; several reload or navigate guest documents. Native input, preload, and webview seams are outside this browser-product conversion. No measurement establishes their peak memory. | + +Thus this contribution reduces a measured high-cost path and other direct product walks; it does not eliminate unbundled loading or certify every remaining test's headroom. Preserve the remaining walks in full validation and use the new diagnostics if they fail. The proposed testing directive is a default for new or changed product acceptances, not a false assertion that all existing tests already comply. + +### Sequencing with PR #9 + +Recommend PR #9 lands first, then integrate its merged main revision before this branch's final affected-file validation and ready review. This branch keeps the theme-reset body with PR #9 and reconciles both the other theme cases and `auth.01.test.ts` against its final implementation. The existing stable-front proxy is sufficient for the proposed restart walks; no new proxy API is assumed. PR #9's currently inspected head `4e85e733` does not list a change to `stable-front-proxy.ts`; if a hook lands there, reuse and review that final API rather than adding a competing one. Proposal/spec/proof work and nonconflicting helper work can proceed meanwhile. This is the recommended merge order for the supervisor to coordinate, not a claim that either PR has been approved to merge. -Collect small resource observations in the parent surface process so that a browser or test-worker crash cannot destroy all the evidence. Cover the canonical planned shard and target paths, including their preparation and teardown failures. The shared browser helper supplements those observations with attempt identity and browser failures. The parent data remains useful if the attempt helper never finishes. +## Failure diagnostics and retention -Retain the following for failed attempts, including attempts that later recover on retry: +The minimum that would have identified this incident is **available RAM, shared memory, free bytes on the effective temporary filesystem, and the Linux OOM-kill counter at surface start and end, plus kernel OOM lines when failure occurs**. Commit to that small set. Read it from the parent surface process so a browser or test-worker crash does not destroy the initial observation. Include the surface preparation/teardown interval in both planned and targeted paths; retain the start record immediately, and mark a missing final observation as unknown after interruption. -- Timestamped available memory and shared memory, available bytes/inodes for the effective temporary filesystem and `/dev/shm`, and initial/peak/final observations from a modest periodic sampler. -- The Linux `oom_kill` counter before and after execution; available cgroup memory-limit/OOM counters; a bounded kernel OOM excerpt when readable. Unavailable probes are explicit, not fabricated zeroes. A system counter increase alone is not attribution to a particular browser. -- Browser version, expected Playwright browser revision, browser crash/disconnect and failed-request error names, and the failed attempt's file/title/project/retry identity. Omit credentials, request query strings, response bodies, and arbitrary process environments. -- On failure, owned process identities and memory/descriptor summaries sufficient to compare with a descriptor limit and correlate kernel PID evidence. Avoid continuous full descriptor scans or permanent NetLog capture. +Keep one resource sidecar per surface execution, linked to the native attempt evidence already recording file/title/project/retry and failures. Retain it when a test attempt fails even if a later retry passes. When available, the shared browser failure hook can add a snapshot at the failed attempt boundary; this is an enhancement, not the minimum contract. Start/end samples provide context and an OOM delta, not a peak or exact allocation-time measurement. A system OOM-counter increase alone cannot attribute a kill to a browser; readable kernel lines provide process identities. If native results are missing, retain the sidecar without inventing an acceptance result. -Prefer one resource sidecar per surface plus small attempt records over a separate monitoring service. Keep it incrementally readable, with explicit truncation/drop counts, and expose retained paths through the normal task/provider reporting. Download the sidecars through the same checked artifact path as native reports. Probe failure produces an observability warning and preserves the original test result. It must neither hide a failure nor create a passing result from missing test evidence. +Read kernel output through a bounded capture and retain at most **64 KiB of relevant OOM lines** for the execution interval, with explicit truncation and unavailable-probe reasons. The two snapshots and bounded excerpt produce at most **128 KiB per surface sidecar and 8 MiB of resource diagnostics per shard attempt**. Enforce these limits when writing; a cap is necessary for bounded retention, but a continuous sampler/ring-buffer framework is not. Retain each surface's small start/end record before using the remaining budget for excerpts. Omit credentials, request query strings, bodies, and arbitrary process environments. Probe failure adds an observability warning and preserves the original test outcome. -The proposed bound is **1 MiB per surface resource sidecar and 8 MiB total resource diagnostics per shard attempt**, retaining initial state, extrema, OOM deltas, and recent failure context when the history is truncated. This cap applies while recording, not only at teardown. Do not add default videos, heap snapshots, or full network logs. Existing trace policy remains unchanged. +Periodic sampling, per-process descriptor inventories, cgroup probes, detailed browser/request event records, and browser-revision comparison remain optional investigation tools. They are not new merge obligations or spec requirements in this contribution. Do not add default videos, heap snapshots, or NetLogs. Existing native retry evidence and trace policy remain unchanged. -On a completed download, the coordinator retains the evidence in its canonical run directory and removes the worker copy. If retrieval is interrupted, bounded diagnostics remain with the stopped attempt until the next safe lease; that lease may replace them after recording that leftover evidence was reclaimed. Ambiguous/live owners prevent reclamation and new dispatch. Thus idle persistent workers do not accumulate an unbounded sequence of diagnostic runs. Coordinator-side `.test-runs/` retention is unchanged. +Write sidecars under the existing checked `.testshards/results` tree, outside disposable scratch, and download them through normal artifact reporting. The worker retains at most the current checkout's bounded resource diagnostics; the existing next-run results reset removes the preceding set, including after interrupted retrieval. Coordinator-side `.test-runs/` retention is unchanged. This does not repair the existing result-download race or promise evidence survives a concurrent checkout reset; that limitation is recorded in the separate follow-up. No new cross-run archive or retention daemon is needed. ## Authority and derivation After proposal convergence, derive the smallest changes in the existing owners: - [`testing-policy.md`](../../specs/arch/testing-policy.md): built application as the default for product browser acceptance when development-module behavior is not under test; worker-owned reusable test runtimes and cleanup expectations. -- [`test-runner.md`](../../specs/arch/test-runner/test-runner.md): resource observation attached to the surface lifecycle, with process teardown preceding disposal of owned temporary files. -- [`blaxel-testshards.md`](../../specs/arch/test-runner/blaxel-testshards.md): attempt temporary directories, exclusive leases across setup/execution/retrieval/cleanup, safe orphan reclamation, and persistent-worker retention. -- [`sharded-execution.md`](../../specs/arch/test-runner/sharded-execution.md) and [`reporting.md`](../../specs/arch/test-runner/reporting.md): resource artifact references, bounded content, unknown/unavailable evidence, and preservation across attempts and downloads. Touch [`preflight.md`](../../specs/arch/test-runner/preflight.md) only where ownership checks actually change its stated contract. +- [`blaxel-testshards.md`](../../specs/arch/test-runner/blaxel-testshards.md): attempt temporary directories, cleanup after process teardown, safe orphan reclamation, and bounded worker-side diagnostic retention. The lease/TTL contract stays intact. +- [`reporting.md`](../../specs/arch/test-runner/reporting.md): minimum resource observations and artifact references, bounded content, unavailable evidence, and retention alongside failed/recovered native attempts. + +Use existing lifecycle and reporting contracts in `test-runner.md` and `sharded-execution.md`; add a cross-reference only if derivation reveals an actual ownership gap. No `preflight.md` delta is planned. Do not turn optional investigation instrumentation into requirements spread across five specs. Update the mirrored proofs after independent spec convergence. Test setup changes may also require proof-only corrections to descriptions of the affected browser walks; they do not change their product promises. Do not prescribe a new framework or adopt unrelated harness internals into spec authority. @@ -141,14 +170,18 @@ Update the mirrored proofs after independent spec convergence. Test setup change All execution is on Blaxel at committed, pushed revisions. The shared development host performs file inspection, edits, Git operations, and lightweight remote coordination only. No tests, builds, browser launches, lint, or type checks have run for this proposal. Apply the test-iteration discipline: red/green at the narrowest relevant file, then affected surfaces, then full verification with the default retry budget. -The proofs should establish a small number of consequential boundaries: reuse and final disposal of real runtime files; partial-build cleanup; preservation of a live job's files while orphaned owned scratch is reclaimed; real environment propagation into Node and Chromium; the result-download/lease boundary; and retained, bounded failure evidence when an attempt fails or its worker exits. Contract coverage handles unavailable resource probes and retention limits. A live targeted Blaxel run and a live planned shard run establish the provider paths. Exercise interruption with disposable owned subprocesses on an isolated Blaxel worker, without intentionally exhausting memory or running repository fixtures restricted to isolated GitHub hosts. +The proofs should establish a small number of consequential boundaries: reuse and final disposal of real runtime files; partial-build cleanup; preservation of a live job's files while orphaned owned scratch is reclaimed; real environment propagation into Node and Chromium; and retained, bounded failure evidence when an attempt fails or its worker exits. Contract coverage handles unavailable probes, unchanged original outcomes, and retention limits. A live targeted Blaxel run and a live planned shard run establish the provider paths. Lease-redesign experiments and result-race repair tests are outside scope. + +Use a dedicated two-worker pool named **`rose-cleanup`**, with workers `tv-testshard-x64-rose-cleanup-01` and `-02`, for cleanup, orphan, and interruption experiments. Explicitly pass `--pool rose-cleanup` through the canonical runner; reduced planned runs also need the documented shard-count override. Exercise interruption with disposable owned subprocesses there, without intentionally exhausting memory or running repository fixtures restricted to isolated GitHub hosts. Never run those experiments on shared `poc` workers. Two workers permit a distinct replacement after a contaminated attempt; they do not guarantee capacity for every retry after repeated contamination, which remains an infrastructure failure with the default retry budget intact. Re-run the converted acceptances without weakening their user-visible assertions. A recorded built-page request inventory should show that they no longer load the unbundled application graph. Validate restart continuity and authenticated navigation directly. Repeated use of one isolated worker should leave no generated runtime directories after cleanup and no growing sequence of resource reports. -`npm run verify -- blaxel` still runs lint/type-check/package-manifest phases on the invoking machine. Run that command from a dedicated Blaxel coordinator host for the final gate; invoking it here would violate the host invariant. Record the exact tested revision, failures, recovered flakes, and any measurement limits. Ready-PR CI remains a merge requirement of the repository, not a substitute for the required Blaxel evidence. +For the final gate, provision a separate sandbox named **`tv-cleanup-coordinator-rose`**, excluded from testshard pool labels. It will fetch the committed, pushed branch and run `npm run verify -- blaxel`, so the invoking machine's lint/type-check/package-manifest phases also execute on Blaxel. The two experimental workers and coordinator are a plan, not currently provisioned capacity. Provision and verify their identities before starting validation; if unavailable, report blocked rather than run anything on this host or silently substitute GitHub execution. The user has already authorized outside-pool Blaxel hosts. + +The experiment allocation uses at most three concurrent sandbox slots beyond the shared 72: **72 + 2 + 1 = 75**. Check actual workspace usage before provisioning/dispatch because other work may already consume that spare capacity; do not claim those slots are reserved or alter shared workers to obtain them. Delete this contribution's experiment workers before the final standard 36-shard verify, leaving only its coordinator alongside the shared pool. That final run uses the normal `poc` lease path without intentional contamination or interruption. Archive its reports before disposing the coordinator. Record the exact tested revision, failures, recovered flakes, and any measurement limits. Ready-PR CI remains the repository's merge requirement; the contribution's execution evidence stays on Blaxel as instructed. Keep browser-version alignment, shard count, browser request limits, retries, and production memory behavior outside this contribution. Forced GC is an investigation tool, not a proposed mitigation. The demo-mode and drag flakes have not been proven to share the resource-exhaustion cause. ## Review status -Proposal: awaiting independent review. No spec, proof, test, or implementation edits yet. The supervisor will receive the pushed proposal commit and draft PR. Subsequent work waits for that review; Josh's spec acceptance remains due at PR time. +Proposal round 1 at `1652787e`: FAIL on lease scope and mixed-version consequences. The [review response](test-harness-resource-cleanup-review-response.md) records dispositions. Revised proposal: awaiting independent re-review. No spec, proof, test, or implementation edits yet. Josh's shared-pool rollout option/timing remains an explicit decision, separate from approval of the code contribution; his spec acceptance remains due at PR time. diff --git a/docs/working/test-harness-resource-cleanup-review-r1.md b/docs/working/test-harness-resource-cleanup-review-r1.md new file mode 100644 index 00000000..5c3043b7 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-review-r1.md @@ -0,0 +1,131 @@ +# Proposal review, round 1: test-harness resource cleanup (draft PR #12) + +**Verdict: FAIL.** Two blockers, both about scope and consequences, not about the diagnosis. The diagnosis is supported, the three recommended fixes are the right ones, and the core `/tmp` answer is safe. The proposal then adds a rework of the worker lease that the cleanup does not need, and it leaves out what happens while other branches keep running the old harness on the same pool. + +- Reviewed: `docs/working/test-harness-resource-cleanup-proposal.md` at `1652787e` on `thopter/test-harness-resource-cleanup`, with the investigation report and both evidence folders. +- Against: Josh's request (the investigator's recommended fixes, and whether `/tmp` on Blaxel workers can be cleaned safely before or after runs), the contribution workflow, the reviewer checklist, and the code and specs the proposal cites. +- I ran nothing. Every check below is a read of files in the checkout. + +## Is the diagnosis supported? Yes + +I checked the report's claims against the retained evidence and the code. + +| Claim | Evidence | Result | +| --- | --- | --- | +| Renderers were killed for lack of memory | Kernel logs: two kills in the shard replay, three in the isolated case, each a `chrome-headless` process | Proven | +| The unbundled page's response buffers are the consumer | The forced-collection experiment released 220 buffers of 2 MiB in each of two renderers; a passing run fell to about 0.55 GB of free memory | Proven for retention; the failing allocation call is inferred, and the report says so | +| The worker's writable filesystem is RAM | Mount table: overlay with its upper layer under `/mnt/tmp`; idle shared memory tracks used root space on both inventoried workers | Supported | +| `getSourceCLIEntry()` leaks one directory per worker process | `test/helpers/product-server.ts:67-76`: the cache is set at line 70, before the build at 71; nothing removes the directory | Confirmed | +| `buildVersionedWebBundle()` leaks one directory per version | `test/helpers/versioned-web-bundle.ts:39-41, 62-70`: no removal, and a failed build stays cached | Confirmed | +| 267 and 146 leaked directories on `poc-131`; 17 and 27 on `poc-03` | Inventory files | Counts and byte totals match | +| The lock is reclaimed on time alone, and released before results are downloaded | `scripts/run-blaxel-testshards.mjs:406` (expiry), `:859` (exit trap), `:457-502` (download after the process ends) | Confirmed | +| The 17 web test files in the conversion table are the `ProductServer.appURL()` users | A search finds exactly those 17, plus the three desktop files the proposal excludes | Confirmed | +| The product runtime lacks the `url-unsupported` view | `test/helpers/product-server.ts:103-132` copies the index, assets, two views and canonical only | Confirmed | + +The report is careful about what it does not know, and the proposal carries those limits forward. + +## Blocker 1: the lease rework is not needed for safe cleanup, and its consequences are not examined + +The proposal says cleanup "therefore extends the existing lease boundary" and calls this "a narrowly required correction". It asks for all of the following: +- the coordinator holds the lease through download and disposal; +- the worker stops unlocking at shell exit; +- reclamation checks the prior supervisor's process identity instead of the expiry time; +- acquisition stays exclusive under racing coordinators; +- an ambiguous owner keeps the worker unavailable; +- results move to an attempt-specific directory. + +**Safe cleanup does not depend on any of that.** The proposal's own directory design already makes it safe: +- A job creates its attempt directory under a random name, so it owns it by construction. It can remove it in the exit path that exists today, before that path releases the lock (`run-blaxel-testshards.mjs:859`). +- Reclaiming an older attempt directory needs one local check: is the recorded supervisor, by process ID and start time, still alive? If it is, leave the directory. That check protects a live job even when the lock has wrongly changed hands. +- Results do not live in the scratch directory, so the download race does not bear on scratch cleanup. + +**The lease weaknesses are real but separate.** +- The download race is a genuine defect: the lock is released at shell exit, and the next job's first step deletes the results directory. +- The expiry concern is weaker than the text suggests. The lock lives 900 or 1200 seconds against a worker timeout of 240 or 480 seconds (`run-blaxel-testshards.mjs:1176-1178`). The Blaxel spec names expiry deliberately as "the final bound" for a worker whose cleanup could not be confirmed (`specs/arch/test-runner/blaxel-testshards.md:29-31`). +- The proposal itself says no observed failure is attributed to these races. + +**The consequences are not handled.** +- *Availability.* Today a worker with unconfirmed cleanup comes back when its lock expires. Under the proposal an ambiguous owner keeps it unavailable. On a pool that every agent shares, that turns a twenty-minute delay into a worker that stays out until someone intervenes. The proposal does not say who clears it or how. +- *Mixed versions.* Each coordinator runs from its own branch and uploads its own shard script. Branches that have not taken this change will keep reclaiming on expiry and unlocking at exit. The new guarantees do not hold while they exist, and the proposal mentions this only for the one-time cleanup. +- *Spec weight.* It replaces a rule the Blaxel spec states on purpose. That is a decision for Josh on its own merits, not a side effect of cleaning `/tmp`. + +**To clear this:** take the lease rework out of this contribution and record the download race as its own item; or keep it, but present it to Josh as a separate decision with the consequences above worked through, and drop the claim that cleanup requires it. + +## Blocker 2: the proposal does not say what happens while other branches still run the old harness + +The helpers and the shard script are per-branch. A run dispatched from a branch without this change still writes `television-source-cli-e2e-*` and `television-web-bundle-*` straight into `/tmp` on the shared workers, and nothing in the new routine cleanup removes them: it deliberately does not infer ownership from a prefix. + +So three statements in the proposal are true only for updated branches: +- "A worker should start its next shard without accumulating the preceding shard's disposable files." +- The legacy removal is a "one-time operation". +- "The normal before/after cleanup covers future runs regardless of which helper created their temporary files." + +Old-branch runs will keep leaking, slowly (about 4 MB per leaked runtime), until every active branch has taken main. That may be perfectly acceptable. Josh should see it and choose, because the alternatives differ in cost: +- accept the residue and repeat the legacy cleanup once branches have caught up; +- let the routine pre-run step remove these two known generated prefixes on a leased idle worker; +- recreate the pool at a chosen time. + +**To clear this:** state the mixed-version period, its effect on each claim above, and the option the proposal recommends. + +## Is the `/tmp` answer safe and evidenced? Yes at its core + +- **"Not blanket" is right and evidenced.** The inventories show the lock, the uploaded shard scripts, the provider's upload directory and the X11 directory all in `/tmp`. Deleting everything before a run would remove the job's own script and lock. +- **"Yes for files the runner owns" is right.** A per-attempt directory with `TMPDIR` pointing into it gives cleanup an exact boundary. It also catches leaks nobody has found yet. The inventories show more leaked prefixes than the two helpers: 222 uploaded shard files, 33 `tv-electron-env-*`, and dozens of `tv-attest-*` directories from test fixtures. +- **The wiring claims hold where I could check them.** Both helpers use `os.tmpdir()`. The Playwright and Chromium claims match my knowledge of those sources, but I did not fetch them. The proposal correctly says a live run must still observe Node paths and Chromium's shared-memory files under the attempt directory. +- **The stated limits are the right ones:** hard-coded `/tmp` paths and the X11 socket directory do not move; deleting a directory does not free memory that a live process still holds. + +The answer would be easier for Josh to act on if it came first and plainly: +- **After a run:** yes, the run removes its own scratch before releasing the worker. +- **Before a run:** yes, for scratch whose recorded owner is dead; never everything. +- **What is already there:** a separate decision (see refinement 2). + +## Is the scope proportionate? Partly + +Apart from the lease rework, the three fixes are the investigator's three recommendations. Their weight in the proposal does not match their effect, and the proposal should say so. + +## Refinements (non-blocking) + +1. **State how much each fix buys.** + - On `poc-131`, `/tmp` held 1.2 GB of 4.3 GB used on a 6.3 GB root. The rest is the checkout and `node_modules` (1.4 GB), the Playwright browsers (0.9 GB) and caches. On `poc-03`, `/tmp` held 0.16 GB of about 3.0 GB. + - The theme-reset test consumed roughly 3.2 GB by itself in a passing run. + - So even a perfectly clean worker runs that unbundled test within about half a gigabyte of the limit. The built-application change removes gigabytes per test; cleanup recovers at most about one, on the worst worker seen. + - Both are worth doing, but the problem statement should give these sizes so the cleanup machinery is sized to its benefit. + +2. **Weigh recreating the pool against the bespoke legacy cleanup.** + - The Blaxel spec already defines provisioning as delete-then-ensure and says it is fresh-only (`blaxel-testshards.md:21-23`). Recreating the pool once clears every worker with no new code. + - The proposal offers that only as a fallback. It prefers a "drained worker under an exclusive maintenance lease" with layout checks, and does not say whether that is new code or an operator procedure. + - Say which, and why it beats recreation. + +3. **Name the minimum diagnostics that would have identified this incident.** + - Free memory, shared memory, free bytes on the temporary filesystem and the kernel's out-of-memory kill counter, read at surface start and end, plus the kernel's kill lines on failure, would have been enough. + - The periodic sampler, the per-process descriptor summaries, the attempt records and the recording-time caps are the investigation's tools. + - The diagnostics section reaches into five specs. Mark the minimum as the commitment and the rest as optional. + +4. **Two validation questions need answers before work starts.** + - *Which pool?* Lease, cleanup and interruption experiments must not run on the shared `poc` pool that other agents use. The proposal says "an isolated Blaxel worker" without naming a pool or its cost against the 75-shard limit. + - *Which host runs the final gate?* The proposal requires `npm run verify -- blaxel` from "a dedicated Blaxel coordinator host". Nothing in the documents shows such a host exists. The workflow accepts ready-PR GitHub CI as full validation, and the Blaxel spec asks only for a live targeted run and a planned-shard run for coordinator changes. Either name the host or use that route. + +5. **Say what stays exposed.** 41 test files load the unbundled page. The proposal converts the 17 that go through `ProductServer.appURL()` and leaves the three desktop ones. About 21 others reach the development page without the product server and keep the same per-load cost. Say whether any of them reload or open several pages, since that is what made theme-reset fail. + +6. **Give the reason for fixing the helpers as well as adding scratch space.** On Blaxel, attempt scratch removed at shard end would clear the helpers' directories without touching the helpers. The helper fix is what protects hosts with no shard script. This development host has 1,388 `television-source-cli-e2e-*` and 727 `television-web-bundle-*` directories in `/tmp` today. That is the argument for doing both, and the proposal does not make it. Whether to clean this host is the owner's call. + +7. **Sequencing with PR #9.** Both change `theme-product-acceptance.test.ts`, and both touch `test/helpers/stable-front-proxy.ts`: PR #9 adds a hook, and this proposal reuses the proxy for restart walks. Say which lands first. + +## Reviewer checklist + +The proposal changes no spec yet, so most checklist items apply at the next stage. Two apply now: + +- **Authority.** The derivation section names the owning specs correctly. With the lease rework removed, `preflight.md` and most of the `blaxel-testshards.md` changes fall away. +- **Testing directives.** The proposed policy line, built application by default for product browser acceptance, is a real human-owned directive and belongs in `testing-policy.md`. It should come with refinement 5's statement of what remains on the development page and why. + +## What I could not establish + +- Whether Blaxel kills a shard process at its timeout. That decides how real the expiry overlap is; I assumed it can fail. +- The make-up of the 0.9 GB of Playwright browsers on the workers, or whether any of the non-`/tmp` baseline is reclaimable. +- The Chromium and Playwright source claims, which I did not fetch. +- Why `poc-32` and `poc-15` started with less free memory than the passing workers. Neither was inventoried, so the share due to `/tmp` there is unknown. + +## Housekeeping + +- Read-only: no edits, commits or pushes, and nothing run on this host or on Blaxel. I listed names in this host's `/tmp` to count the leaked directories; nothing was changed. +- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md new file mode 100644 index 00000000..d95068f1 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -0,0 +1,28 @@ +# Resource-cleanup proposal: round 1 response + +Reviewed proposal: `1652787e`. Independent reviewer: Claude Fable (Nina), arranged by the supervisor. [Full review](test-harness-resource-cleanup-review-r1.md): FAIL on two blockers. This response changes proposal documents only; no spec, proof, or implementation work has started. + +## Blockers + +| Finding | Assessment and disposition | +| --- | --- | +| B1: lease redesign is unnecessary for safe scratch cleanup and leaves availability/mixed-version consequences unresolved | Accepted. Remove the lease redesign, indefinite unavailability rule, and report-root migration. Keep the current TTL and release behavior. Private attempt directories and process-identity checks govern disposal independently of leases. Unknown ownership means retain that directory, not permanently quarantine the worker. Record the real result-download race as a separate follow-up, including the current timeout/TTL bounds and consequences a future proposal must address. No decision to replace TTL is being made for Josh. | +| B2: old branches continue leaking outside the new directory | Accepted. Restrict cleanup claims to runs carrying the change. Explicitly describe continuing old-branch residue and why an initial cleanup may need repetition. Recommend tolerating residue during migration, then a coordinated drained-pool recreation using existing provisioning. Present alternatives and leave the shared-pool option and timing to Josh. That operation is neither performed nor silently authorized by accepting the code proposal. | + +## Refinements + +| Finding | Assessment and disposition | +| --- | --- | +| R1: quantify relative benefit | Accepted. Add the roughly 3.2 GB test-induced shared-memory increase and low headroom from the passing experiment, versus 1.16 GB / 0.079 GB of the two leaked prefixes on inspected workers. Keep the expected saving from built loading distinct from a measured post-fix saving. Intentional checkout/browser/cache occupancy remains outside cleanup. | +| R2: compare pool recreation with legacy deletion | Accepted. Prefer existing drain/delete/ensure over bespoke legacy-deletion code. The alternative prefix removal would be an operator procedure on explicitly drained workers, not routine pre-run behavior. Josh owns the shared-pool disruption and timing. | +| R3: commit only to minimum diagnostics | Accepted. Require parent start/end RAM, shared-memory, temporary-free-space and OOM-counter observations plus bounded kernel OOM lines on failure, linked to existing native attempt reports. Retain failures recovered by retry. Continuous sampling, descriptor inventories and other detailed probes are optional investigation tools. Fixed-size snapshots and a bounded excerpt satisfy retention without a monitoring framework. Reduce derivation to the relevant existing spec owners. | +| R4: name validation infrastructure and account for capacity | Accepted. Plan two private `rose-cleanup` workers for cleanup/interruption experiments and `tv-cleanup-coordinator-rose` for the final gate's invoking-machine phases. They are not yet provisioned or reserved. At most three additional slots fit the documented 72+3=75 bound only if no other work consumes that headroom; check usage first. Remove the private workers before a standard final verify on the shared pool. No fallback to this host or substitution of GitHub execution: the user's Blaxel-only instruction takes precedence over the suggested CI alternative. | +| R5: disclose remaining unbundled exposure | Accepted. Add a named inventory and concrete reload/multiple-page cases: telemetry, navigation setup, authentication reconnect, and Electron theme resets. Source inspection found 38 test files with a literal app-entry reference plus two indirect helper consumers; 41 includes their shared helper. This minor counting distinction does not change the finding: roughly two dozen test files remain exposed. Explain the in-process server and Electron boundaries without declaring them inherently dependent on Vite or claiming adequate headroom. | +| R6: justify helper fixes in addition to scratch | Accepted. Helper teardown frees within-shard worker outputs and protects runners with no Blaxel wrapper. Attribute this host's directory counts to the independent review; no local cleanup is part of this work. | +| R7: sequence PR #9 | Accepted with current-head qualification. Recommend #9 lands first, then integrate its final changes before affected-file validation and ready review. Keep its theme-reset ownership. The inspected #9 head `4e85e733` changes theme/auth files but does not list `stable-front-proxy.ts`; do not assume an unmerged proxy hook exists. Reconcile the final helper API if it changes. The supervisor coordinates merge order. | + +## Decision still needed from Josh + +Choose the shared-pool migration option and timing. Recommendation: tolerate old-branch residue temporarily, have active branches take the cleanup change, then drain and recreate the pool. This buys simplicity at the cost of temporary reduced headroom, a coordinated interruption, and cold caches. An earlier recreation may be justified by capacity but can be contaminated again by old branches. Proposal convergence and code work can proceed without performing this operation; pool-wide cleanliness cannot be claimed until it happens and leaking old runs stop. + +Validation of this revision consists of read-only code/policy/PR inspection and document consistency/whitespace checks. No tests, builds, browsers, or servers were run on this host, and no remote experiments or cleanup were performed during remediation. The supervisor will receive the pushed revision for independent re-review. From d3570bf4a305c7fb0b6cde40e3fa07e90dc0baf0 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:37:59 +0000 Subject: [PATCH 03/64] Revise resource cleanup proposal around worker lock decisions --- docs/working/blaxel-results-download-race.md | 21 ---- .../test-harness-resource-cleanup-proposal.md | 118 +++++++++++------- ...arness-resource-cleanup-review-response.md | 44 ++++--- 3 files changed, 103 insertions(+), 80 deletions(-) delete mode 100644 docs/working/blaxel-results-download-race.md diff --git a/docs/working/blaxel-results-download-race.md b/docs/working/blaxel-results-download-race.md deleted file mode 100644 index ba367eb7..00000000 --- a/docs/working/blaxel-results-download-race.md +++ /dev/null @@ -1,21 +0,0 @@ -# Blaxel result retrieval can race the next checkout - -Follow-up identified while investigating Chromium resource failures. This item is outside draft PR #12's cleanup implementation. No observed Chromium failure has been attributed to this race. No change to the lease protocol is authorized by this document. - -## Evidence and consequence - -At `b5477211`, `scripts/run-blaxel-testshards.mjs` does the following: - -1. The remote shard script's `EXIT` trap releases `/tmp/tv-testshard.lock` at shell exit. -2. `runShard()` waits for that remote process to finish, then downloads the summary and artifacts from `/workspace/television/.testshards/results`. -3. A new lease can start another checkout, whose preparation removes `.testshards`, before the prior coordinator finishes downloading. - -The source establishes an unprotected interval, not an observed collision. A collision could lose the previous job's evidence or make it encounter another job's files. Existing report identity validation remains necessary but cannot recover deleted evidence. - -## Scope for a separate proposal - -Assess retaining the existing lease through retrieval versus using immutable attempt-specific report locations, with bounded orphan retention in either case. Work through coordinator interruption, download failure, worker reuse, and compatibility with older branches that still unlock at shell exit and reset the fixed results root. Compare the smallest correction before adding lease machinery. - -TTL recovery is a separate availability decision. The current lock lives 900/1200 seconds against a worker timeout of 240/480 seconds; the Blaxel spec deliberately leaves uncertain cleanup to TTL as a final bound. Whether the provider can leave a supervisor alive beyond that timeout remains unproven. Replacing TTL with indefinite quarantine would require Josh to choose its availability cost and an explicit operator recovery procedure. Scratch ownership checks do not require that policy change. - -The cleanup proposal preserves the existing results location and lease behavior. Its diagnostics inherit this download-race limitation. This follow-up has no assigned implementation or rollout date. diff --git a/docs/working/test-harness-resource-cleanup-proposal.md b/docs/working/test-harness-resource-cleanup-proposal.md index 8cc32ff3..7df17f63 100644 --- a/docs/working/test-harness-resource-cleanup-proposal.md +++ b/docs/working/test-harness-resource-cleanup-proposal.md @@ -1,16 +1,16 @@ # Test-harness resource cleanup -Proposal for independent review. Owner: rose. Base: `origin/main` at `b5477211`. Branch: `thopter/test-harness-resource-cleanup`. The supervisor arranges independent review of the proposal, specs, proofs, and implementation; Josh reviews every spec delta at PR time. No production behavior changes are proposed. +Proposal for independent re-review, revised 2026-10-27 following Josh's scope decisions. Owner: rose. Branch: `thopter/test-harness-resource-cleanup`, draft PR #12, originally based on `b5477211`. Current `origin/main` was inspected at `a8e73fbf`; it has not yet been integrated into this branch. The supervisor arranges independent review of the proposal, specs, proofs, and implementation; Josh reviews every spec delta at PR time. No production behavior changes are proposed. ## Problem and intended result -Repeated unbundled application loads can consume several GiB of Chromium response-buffer memory. Persistent Blaxel workers also retain temporary test runtimes across runs, reducing the memory and filesystem capacity available to browsers. The [investigation](chromium-insufficient-resources.md) establishes renderer OOM kills and reproduces `ERR_INSUFFICIENT_RESOURCES`; the exact failing allocation syscall remains unobserved. +In the investigated runs, repeated unbundled application loads consumed several GiB of Chromium response-buffer memory. Persistent Blaxel workers also retain temporary test runtimes across runs, reducing the memory and filesystem capacity available to browsers. The [investigation](chromium-insufficient-resources.md) establishes renderer OOM kills and reproduces `ERR_INSUFFICIENT_RESOURCES`; the exact failing allocation syscall remains unobserved. -The fixes have different effects. The theme-reset walk added roughly 3.2 GB of shared memory in a passing run that reached about 0.55 GB available RAM. Switching that walk to the built application removes its hundreds of development-module requests per load; the expected memory saving is large, but still needs a post-change measurement. By comparison, the two leaked runtime prefixes occupied 1.16 GB on the worst worker inspected and 0.079 GB on the other. All of `/tmp` occupied 1.20 GB and 0.16 GB respectively. Cleanup recovers accumulated capacity; it does not make repeated unbundled loading cheap or guarantee enough headroom. The original worker also retained roughly 1.4 GiB of checkout/dependencies, 0.9 GiB of browser installations, and other intentional caches. Those are outside this cleanup. +The fixes have different effects. The historical theme-reset walk added roughly 3.2 GB of shared memory in a passing run that reached about 0.55 GB available RAM. Built loading removes hundreds of development-module requests per load, but those historical numbers cannot predict its incremental saving after the Vite response correction described below. By comparison, the two leaked runtime prefixes occupied 1.16 GB on the worst worker inspected and 0.079 GB on the other. All of `/tmp` occupied 1.20 GB and 0.16 GB respectively. Cleanup recovers accumulated capacity; it does not guarantee browser headroom. The original worker also retained roughly 1.4 GiB of checkout/dependencies, 0.9 GiB of browser installations, and other intentional caches. Those are outside this cleanup. -This contribution fixes the cached runtime lifetimes, runs suitable product acceptances against the built application, and retains compact resource evidence when an acceptance fails. For runs using the updated harness, owned disposable files should be removed after confirmed process teardown, with orphan cleanup on a later run when teardown could not finish. Older branches will continue to leave unowned files until updated. Cleanup must preserve active jobs, provider services, deliberate dependency caches, and reports outside scratch space. +This contribution fixes the cached runtime lifetimes, runs suitable product acceptances against the built application, and retains compact resource evidence when an acceptance fails. It also makes the worker reservation cover preparation, execution, process teardown, and result download. After taking that lock and confirming the worker is idle, each run removes the two known leaked-runtime prefixes and stale run scratch. This repeatedly reclaims residue from older test revisions as well as newer ones. Cleanup preserves active jobs, provider services, deliberate dependency caches, and undownloaded reports. -PR #9 owns the built-application conversion of **“executable theme reset reloads connected Chromium documents under confirmed state.”** This branch leaves that test body to PR #9. Its existing failure is useful investigation evidence, not an obligation to duplicate that fix here. +PR #9 merged as `f81a2465` on 2026-10-01 and owns the built-application conversion of **“executable theme reset reloads connected Chromium documents under confirmed state.”** Reuse that change when integrating main; do not duplicate it. Main also includes `982e564d` (PR #32), making registry Vite services answer conditional requests in full. Its runner spec records Chromium 145 retaining about 2 MB per 304 response, roughly 465 MB per application reload. That newer mitigation changes the baseline for future measurements. The investigation numbers above describe the historical runs, not current main's residual memory cost; neither the helper leak nor the lock/download gaps is repaired by the Vite response change. ## Evidence available before implementation @@ -35,27 +35,47 @@ There are two proven helper leaks: Other `/tmp` entries include Playwright artifacts and transform caches, Node's compile cache, `tsx` caches, Chromium temporary files, X11 state, an `xvfb-run` authority directory, the Git askpass helper, hundreds of uploaded shard scripts, test-runner fixtures, and `multipart-uploads`. The idle process tables include Blaxel's `sandbox-api`, `ukp-fs`, and entrypoint processes. The precise owner of every miscellaneous entry was not established; in particular, the upload directory must be treated as outside Television's disposal ownership. Neither idle snapshot had an open `/tmp` descriptor or cwd reference, which establishes only those instants. -## Can we clean `/tmp` safely? +## Worker exclusion, result retrieval, and safe `/tmp` cleanup -**After a run:** yes, remove that run's own scratch after its processes stop and before the existing exit path releases the worker. **Before a run:** yes, reclaim owned scratch only when its recorded supervisor and descendants are confirmed gone. **Existing unowned files:** require a separate rollout decision from Josh, described below. Never delete all of `/tmp`: the worker lease itself lives there, as do provider and display infrastructure. Name prefixes or age alone do not establish that a file is unused. +Josh has selected routine cleanup under an exclusive worker lock. There is no pool drain-and-recreate plan or outstanding rollout choice. Before each run, remove only the two known legacy prefixes and stale scratch in the dedicated runner namespace; after execution, remove that attempt's scratch only after its processes stop and its evidence has been finalized outside scratch. Never delete all of `/tmp` or use age alone as proof of disuse. -### Concurrency and the present lease boundary +### What the existing code establishes -The canonical coordinator intends one shard job per sandbox. [`acquireAny()` and `lockCommand()`](../../scripts/run-blaxel-testshards.mjs) acquire `/tmp/tv-testshard.lock`; another coordinator skips an unexpired lock. A normal broad run uses separate workers for its shards. Within a shard, surface tasks execute sequentially, but a surface can have several test workers and many child processes. For example, the browser-app surface uses two Playwright workers on CI; browser-app-real-stack uses one. +The canonical coordinator intends one shard job per sandbox. [`acquireAny()` and `lockCommand()`](../../scripts/run-blaxel-testshards.mjs) acquire `/tmp/tv-testshard.lock`; another coordinator skips an unexpired lock. Within a shard, surface tasks execute sequentially, but a surface can have several test workers and many child processes. This does not prove that a worker never has overlapping jobs: -This is not evidence that a sandbox can never have overlapping jobs: +1. Lock expiry allows reclamation based on time alone. The record identifies the coordinator host/PID, not the remote shard supervisor. Setup is not fully covered by surface owner tokens. The configured TTL is 900/1200 seconds against a 240/480-second worker timeout, but provider timeout is not evidence that every descendant has stopped. The current spec explicitly uses TTL as the final availability bound for uncertain cleanup. +2. The shard shell's `EXIT` trap releases the lock before `runShard()` downloads results from `/workspace/television/.testshards/results`. A following checkout clears that fixed tree. This leaves an unprotected interval in which the previous job's evidence can disappear or be replaced. Report identity checks can detect some mismatches but cannot restore deleted files. This source-level race is present at both the branch base and inspected main; no collision was observed during the Chromium investigation. +3. Direct Blaxel process API calls bypass this repository lock. The inventory used that API. `manage-blaxel-testshards.mjs` also provisions and checks workers through `process.exec`, and deletes them through the provider API. The idle snapshots establish neither provider serialization nor a history-wide absence of overlap. -1. Lock expiry currently allows reclamation based on time alone. The lock records the coordinator's host/PID, not the remote shard supervisor's live identity. Setup processes are not all covered by surface owner tokens. The stale-owner reaper deliberately leaves live-supervisor tokens alone and can exit successfully with `liveOwners` present. However, the configured TTL is 900/1200 seconds against a 240/480-second worker timeout. Expiry is deliberately the spec's final availability bound when cleanup cannot be confirmed. Whether the provider can leave a job alive past that bound was not established. -2. The shard shell's `EXIT` trap releases its lock before `runShard()` downloads results from the fixed `/workspace/television/.testshards/results` directory. A following job can reset those files while the prior coordinator reads them. -3. Direct Blaxel process API calls are not serialized by this repository lock. The read-only inventory itself used that API. The inspection found no overlapping test jobs, but does not establish a history-wide absence of overlap. +These are demonstrated protocol gaps, not demonstrated causes of the Chromium failures. The download-race follow-up is incorporated here; both its correction and the removal of time-based reclamation belong in PR #12. -**The existing lease protocol, TTL, and result location remain unchanged in this contribution.** Safe scratch cleanup uses a fresh private directory and that directory's process ownership, independently of whether the lease has changed hands. A live or uncertain owner prevents removal of that directory; it does not add a permanent worker quarantine. Existing stale-process attribution and TERM/KILL cleanup remain in force. Cleanup cannot make overlapping checkouts safe and does not claim to do so. +### One lock through process teardown and download -The result-download race is recorded as a [separate follow-up](blaxel-results-download-race.md). Resolving it, or replacing TTL recovery with operator intervention, needs its own scope and consequences assessed, including compatibility with older coordinators. Neither is required for this scratch cleanup. No observed Chromium failure has been attributed to these lease weaknesses. +Josh's calibration is that only our runner and pool scripts touch these workers. Correct those code paths using the existing lock; do not add a worker controller, persistent compatibility fence, access-control system, or defenses against hypothetical API callers. + +Retain the worker lock until both conditions hold: **all execution for the attempt is stopped, and its results are downloaded and validated at the coordinator.** Keep atomic directory acquisition and the existing `lockId` ownership check. Write new held locks with `expiresAt: 0`: the inspected older runner reclaims only a positive timestamp in the past, so it also leaves these locks alone. Remove the new runner's age-only reclamation. Timeouts still bound execution and trigger cleanup, but they never authorize another run to overwrite a live attempt or its undownloaded results. The directory is removed on confirmed release, as it is today; it does not become a permanent worker installation. + +The runner's order becomes: + +1. Acquire the lock, then confirm the worker has no prior test/shard work before removing known leftovers. Extend the existing owner-token attribution to the outer shard, including checkout and setup children. Record the remote process name and PID/start identity so recovery can distinguish the run from PID reuse; the coordinator's host/PID alone is insufficient. +2. Prepare and execute the shard under that lock. Keep the existing all-shards-acquired barrier, sequential surfaces, and attempt/retry behavior. Record that dispatch was requested before making the API call. If its outcome is unknown, retain the lock until execution status and cleanup are confirmed. +3. On success, failure, timeout, or interruption, stop and audit the run's attributed processes, including descendants of an exited shell. Reuse the existing owner audit and TERM/KILL cleanup. A shell exit or successful kill API response alone is insufficient. The shell finalizes reports but no longer releases the lock in its `EXIT` trap. +4. Download the existing summary, native reports, logs, setup/cleanup evidence, and new diagnostic sidecars to the coordinator's run directory. Validate the existing run/attempt identities and required files. Missing or corrupt evidence remains an infrastructure failure; an early setup failure records which reports were never produced and retrieves what is available. +5. Release the matching lock only after teardown and retrieval complete. Keep the fixed checked results tree: protecting it through download closes the checkout-reset race without a separate artifact archive. Release unused locks after acquisition/upload failures only when execution was never requested and there are no pending writes or produced results to retain. + +All runner exit paths must follow that order, including acquisition rollback, architecture rejection, both signal paths, retry, and exception cleanup. An interrupted download retains the lock; retry retrieval without repeating the tests. A failed teardown or uncertain dispatch excludes that worker from retries. Keep the default retry budget. + +### Recovery, old branches, and pool operations + +A coordinator crash can leave a lock indefinitely. Report the worker, lock/run identity, last phase, and paths needed for recovery. Once the original coordinator is confirmed ended, recovery reuses the same stop/audit, download, and matching-lock release steps. If execution status, process teardown, or retrieval cannot be confirmed, leave the lock held and report the specific reason. No timer or blind `rm -rf` is a recovery procedure. This availability cost is explicit; no heartbeat service or general ownership-transfer protocol is proposed. + +Older branches matter in two concrete ways. Their helpers still leave the two known directory prefixes, which every updated run removes. Their coordinators also unlock at shell exit and use timestamp expiry. New non-expiring locks stop the inspected old acquisition code from taking an updated run's worker, but cannot correct an old coordinator's own early release. Carry the small lock lifecycle correction into branches still dispatching to the shared pool, and let already-started old runs finish their downloads before relying on the corrected behavior. There is no permanent old-client exclusion mechanism or separate controller/test-revision architecture. An existing legacy lock, even expired, is not reclaimed merely because of its age; uncertain ownership needs the same recovery checks. No pool drain/recreation is required to reclaim test residue. + +`manage-blaxel-testshards.mjs` is the other actual API caller to correct. Its mutation commands must respect the same lock: obtain locks on the affected existing workers before deletion and refuse busy workers, releasing unused locks if acquisition fails. Fresh provisioning runs with dispatch to that pool stopped until setup and architecture verification finish. Listing/status is read-only. This covers our pool script without adding a generic API gateway or maintenance framework; manual pool operations use the same drained-pool discipline. ### A directory for each shard attempt -Give every leased shard **attempt** a fresh, short, private directory beneath a runner-owned root, for example `/tmp/tv-testshards//`. Record its run, shard, attempt, lease identity, and remote supervisor PID/start identity outside its disposable `tmp/` child. Record ownership before using scratch; the pre-dispatch directory must not be mistaken for a dead supervisor. Upload/dispatch failure removes only the directory this coordinator created when execution is known not to have started. An unknown dispatch outcome leaves it for confirmed recovery. Use a short path because browser and Electron processes can create Unix sockets with limited pathname length. +Give every locked shard **attempt** a fresh, short, private directory beneath a runner-owned root, for example `/tmp/tv-testshards//`. Record its run, shard, attempt, lock identity, and remote supervisor identity outside its disposable `tmp/` child. Record ownership before using scratch; a pre-dispatch directory must not be mistaken for an abandoned run. Upload/dispatch failure removes only this attempt's directory when execution is known not to have started and pending requests have settled. An unknown outcome leaves the lock and scratch for recovery. Use a short path because browser and Electron processes can create Unix sockets with limited pathname length. Export an absolute `TMPDIR` pointing to that child before running preparation and test commands. Keep scripts, plans, askpass, and setup sidecars within the attempt directory as well. Preserve `TMPDIR` through child environments. Continue to keep deliberately persistent npm, Electron, browser-download, and dependency-hash caches at their existing locations outside scratch space. Playwright's transform cache may become attempt-local initially; measure any resulting setup cost before deciding it deserves a separately bounded persistent cache. @@ -67,28 +87,18 @@ The mechanism is supported by both source and installed dependencies: This establishes the intended wiring; a live Blaxel test must still observe Node-created paths and actual Chromium shared-memory descriptor paths under the attempt directory. We have not run that check at the proposal stage. Setting `TMPDIR` cannot relocate hard-coded `/tmp` paths or X11's global socket directory; the runner's explicit temporary paths must be moved deliberately, and external infrastructure remains untouched. Chromium also unlinks shared-memory files while holding them open, so directory deletion cannot substitute for stopping the owning processes. -Cleanup has two paths: - -1. **Before an attempt:** reclaim earlier directories in the dedicated namespace only when recorded PID/start identities are dead and no attributed descendants remain. Revalidate before removal and reject unexpected directory/symlink layouts. Missing metadata or unreadable process evidence means skip and report, not guess. A live supervisor protects its directory even if its lease has expired. Create the new attempt's directory fresh; never delete a guessed path to make room. -2. **After execution:** finalize evidence in the existing results directory outside scratch, complete process teardown, then remove this attempt's scratch from the existing exit path before it releases the lock. Setup failure, test failure, timeout, and interruption follow the same ownership rule. The coordinator's existing unconfirmed-execution path removes scratch only after its process cleanup is confirmed. Otherwise leave it for a later ownership-checked reclamation. Preserve the original outcome if cleanup itself fails, and report the leftover path. - -The supervisor identity alone is not enough if detached descendants survived it. Reuse the existing owner-token audit and attach the attempt identity to setup children as needed for attribution; do not kill unrelated processes to make a directory reclaimable. A missing/uncertain owner can leave residual scratch requiring operator attention, but does not change lease availability. This is a disposal safety rule, not a promise that every abandoned byte can always be reclaimed automatically. +### Exact deletion scope and order -Reports stay under the current checked results root and are downloaded normally. Scratch cleanup does not remove them. This retains the known download race as an explicit limit instead of redesigning report storage inside this contribution. +Before preparation for **every run**, while holding the worker reservation and after confirming no prior execution or retrieval remains, remove: -### Mixed versions and existing unowned files: Josh's rollout decision +- Real directories directly under `/tmp` whose basenames match `television-source-cli-e2e-*` or `television-web-bundle-*`. The helper source and inventories establish these as leaked test builds; Josh has authorized their recurring removal. Missing per-directory metadata on these legacy copies is expected. The whole-worker exclusion and idle check supply the disuse evidence that the prefix alone cannot supply. +- Stale attempt directories beneath the dedicated `/tmp/tv-testshards/` root, identified by runner metadata and confirmed completed/recovered ownership. Exclude the current reservation's staging directory. A dead supervisor alone does not prove that descendants or delayed dispatch are gone. Missing or inconsistent metadata needs recovery, not an age-based guess. -Every coordinator uploads the script from its own branch. Older branches keep creating `television-source-cli-e2e-*` and `television-web-bundle-*` directly under `/tmp`, bypass the new attempt directory, and will keep leaking them. One cleanup now cannot end that accumulation. The updated harness cleans files created under its own scratch, including helpers not individually repaired; it cannot make that claim for old-branch runs or hard-coded external paths. +Revalidate directory identity immediately before deletion; reject symlink roots, unexpected mount boundaries, or paths escaping the exact allowed parent. Recursive removal must not follow nested links to another tree. Preserve the lock directory, provider uploads, X11 state, unrelated temporary files, checkouts, deliberate caches, and every undownloaded report. Do not broaden the legacy patterns to other Television-looking names. Report per-prefix counts, allocated bytes removed, filesystem free space before/after, and skips or errors; those two space measurements need not be equal when files remain open or the filesystem accounts storage differently. -**Recommendation for Josh's decision:** accept temporary residue while active branches take the change, then arrange a drained-pool recreation using the existing delete-then-ensure procedure. This needs a coordinated pause in dispatch and pays cold provisioning/dependency costs, but removes all old test residue with existing machinery. If capacity becomes a problem sooner, Josh can schedule an earlier recreation knowing that old branches can contaminate the fresh workers again. No date or shared-pool deletion is authorized by this proposal. +After execution, finalize evidence outside scratch, stop all attributed processes, then remove this attempt's scratch while the reservation remains held through download. Setup failure, acceptance failure, timeout, and interruption use the same ownership rule. If teardown is uncertain, preserve scratch and keep the worker reserved. If deletion fails after confirmed teardown, preserve the original test outcome and add cleanup failure evidence; the worker cannot start another test until pre-run cleanup succeeds. Evidence must still be downloaded. A partial deletion must never be reported as reclaimed capacity for the whole directory. -| Option | Benefit | Cost and consequence | -| --- | --- | --- | -| Accept residue during migration, then drain and recreate the pool (recommended) | No new legacy-deletion code; clears every worker and every old disposable prefix. | Temporary reduced headroom, a coordinated interruption, and cold caches. May need repetition if old branches continue running. | -| Operator removes the two known prefixes from explicitly drained workers | Preserves dependency and browser caches. | Must establish that no job or other consumer uses those files; only recovers these two prefixes; needs repetition during mixed versions. This would be an operator procedure, not routine prefix-based deletion in this PR. | -| Add routine legacy-prefix cleanup | Reclaims old-branch leaks repeatedly. | Adds deletion without recorded ownership, dependent on proving the whole worker idle despite mixed coordinators; disproportionate to this fix and not recommended. | - -Josh must choose the shared-pool rollout option and timing. The contribution can be reviewed and implemented without doing that operation; it cannot claim the whole pool is clean until the decision is executed and old branches stop leaking. No shared-pool or development-host cleanup has been performed. +Unknown ownership or a live consumer means skip deletion **and exclude that worker from test dispatch**, with a recoverable infrastructure failure. There is no promise that every abandoned byte is automatically reclaimable, nor a fallback to deleting unrelated files to obtain headroom. Known residue left by old test revisions is removed on subsequent updated runs without recreating workers or losing their caches. This proposal revision performs no worker cleanup or configuration changes. ## Worker lifetime for cached runtimes @@ -102,6 +112,16 @@ The helper fix matters even with shard scratch: it releases files between test w ## Built application for product acceptances +### Assessment after PR #32 + +**Keep the conversion set below, for product fidelity and fewer application-module requests; do not present it as necessary to repair the measured 304-buffer leak.** PR #32 (`982e564d`, merged 2026-10-09) removes conditional request headers at the registry Vite service boundary, so those services return full responses. Its recorded Chromium 145 evidence identifies the 2 MB buffer retained per 304 until context closure. That change directly addresses the repeated-reload mechanism for tests using those services, including tests that remain on Vite. Merely counting unbundled modules no longer establishes that the same accumulation persists on current main. + +The listed acceptances assert shipped product behavior and already use a real `ProductServer` with built assets available. Loading those assets exercises the packaged application, asset routing, and authentication/reconnect behavior users receive; a development-module page can miss packaging and delivery mistakes. It also avoids hundreds of module requests per application load, including in multi-page/restart walks. Fewer requests is a concrete change, but its incremental memory and time benefit after #32 is unmeasured. Do not claim a specific reduction, that #32 is insufficient, or that all remaining Vite tests are resource-unsafe. + +The conversion cost stays limited to this existing product-server set and the stable-origin restart helper. Preserve the assertions and #9's merged theme-reset conversion. Keep #32 for the remaining Vite services and keep development loading for module/fixture seams. Converting the in-process-server and Electron suites would require broader harness work and is still deferred. Validate the converted paths functionally and confirm their request inventory; any before/after resource comparison must use a baseline containing #32. This is a recommendation based on the tests' intended boundary and source inspection, not a post-#32 memory experiment. + +### Conversion boundaries + Use the already-built web application served by `ProductServer`. Existing registry pre-commands build the web and artifact documents before both browser-app surfaces, and `installProductRuntimeAssets()` already copies the application index and assets. Preserve the tests' assertions, real browser behavior, CSS-motion declarations, authentication, and server lifecycle. The proposed conversion set is the browser product walks using `ProductServer.appURL()`: @@ -130,17 +150,19 @@ At `b5477211`, a read-only inventory finds 38 e2e test files with a literal `/pa | --- | --- | | Web `telemetry-session.test.ts` | One case reloads the application; another opens two tabs in one persistent context, closes it, then opens a new context. This is a concrete remaining multi-page exposure. It uses an in-process server with an injected telemetry sink; converting that boundary needs its own proof review. | | Web `artifact-navigation.01.test.ts`, `artifact-navigation-history.test.ts` | Their `beforeEach` loads the development app, clears storage, then reloads it: two full graph loads per case. They use directly controlled `Store`/`Server` instances. These are sensible next conversion candidates, not proven-safe exceptions. | -| Web `auth.02.test.ts` | The silent reconnect case explicitly reloads the app. Other cases inject authentication failures into an in-process server. PR #9 also changes this file, so reassess its final walks after that lands. | +| Web `auth.02.test.ts` | The silent reconnect case explicitly reloads the app. Other cases inject authentication failures into an in-process server. Reconcile this base-revision inventory against PR #9's merged changes before implementation. | | Web `reload-after-reconnect.01/.02.test.ts`, `guerilla-artifact-sharing.test.ts`, `guerilla-artifact-sharing-resilience.test.ts` | These open one application page and reload/recover artifact documents or reconnect the event socket. The reconnect continuity case explicitly expects no top-level navigation. They control server internals/producer-consumer seams; an artifact reload is not evidence of a full application graph reload. | | Web `settings.test.ts`, `platform-marker.test.ts`, `artifact-navigation.02.test.ts`, `appearance-delivery.test.ts`, `theme.test.ts`, `artifact-full-screen.test.ts` | The inspected source-app cases use one application page. Other cases in these files use fixtures or built assets. Retain their in-process server, platform, appearance, and navigation boundaries for this contribution; no claim that development modules are inherently necessary. | | Desktop `all-webview-dispatcher.01/.02.test.ts`, `webview-event-parity.test.ts` | The executable-theme parity walk explicitly waits for a top-level document reset and later closes/relaunches Electron. The navigation-history walk quits and relaunches. These remain exposed; application resets, fresh Electron processes, and guest-webview reloads must be distinguished in any follow-up conversion. | | Desktop `artifact-view-webview-wheel.test.ts`, `guerilla-artifact-sharing.test.ts`, `path-webview-content-reload.test.ts`, `artifact-navigation.test.ts`, `native-navigation-key.test.ts`, `artifact-navigation-external.test.ts` | Launch the development application in Electron; several reload or navigate guest documents. Native input, preload, and webview seams are outside this browser-product conversion. No measurement establishes their peak memory. | -Thus this contribution reduces a measured high-cost path and other direct product walks; it does not eliminate unbundled loading or certify every remaining test's headroom. Preserve the remaining walks in full validation and use the new diagnostics if they fail. The proposed testing directive is a default for new or changed product acceptances, not a false assertion that all existing tests already comply. +Thus this contribution moves suitable product walks onto shipped assets and reduces their module requests; it does not establish a post-#32 memory saving, eliminate unbundled loading, or certify every remaining test's headroom. Preserve the remaining walks in full validation and use the new diagnostics if they fail. The proposed testing directive is a default for new or changed product acceptances, not a false assertion that all existing tests already comply. + +### Reconciliation with current main -### Sequencing with PR #9 +PR #9 has merged. Integrate current main before deriving the affected spec/proof changes and implementation; preserve its theme-reset conversion and reconcile the other theme cases, authentication walks, and stable-front proxy with the actual merged API. The inventory above is explicitly tied to `b5477211`, not a claim that all listed lines remain on today's main. Do not convert an already converted case again. -Recommend PR #9 lands first, then integrate its merged main revision before this branch's final affected-file validation and ready review. This branch keeps the theme-reset body with PR #9 and reconciles both the other theme cases and `auth.01.test.ts` against its final implementation. The existing stable-front proxy is sufficient for the proposed restart walks; no new proxy API is assumed. PR #9's currently inspected head `4e85e733` does not list a change to `stable-front-proxy.ts`; if a hook lands there, reuse and review that final API rather than adding a competing one. Proposal/spec/proof work and nonconflicting helper work can proceed meanwhile. This is the recommended merge order for the supervisor to coordinate, not a claim that either PR has been approved to merge. +Main also now owns full responses from registry Vite services and selection of a contributor's origin repository for remote checkout. Preserve both. Measure built-app memory savings against this new baseline rather than attributing PR #32's benefit to PR #12. ## Failure diagnostics and retention @@ -152,17 +174,17 @@ Read kernel output through a bounded capture and retain at most **64 KiB of rele Periodic sampling, per-process descriptor inventories, cgroup probes, detailed browser/request event records, and browser-revision comparison remain optional investigation tools. They are not new merge obligations or spec requirements in this contribution. Do not add default videos, heap snapshots, or NetLogs. Existing native retry evidence and trace policy remain unchanged. -Write sidecars under the existing checked `.testshards/results` tree, outside disposable scratch, and download them through normal artifact reporting. The worker retains at most the current checkout's bounded resource diagnostics; the existing next-run results reset removes the preceding set, including after interrupted retrieval. Coordinator-side `.test-runs/` retention is unchanged. This does not repair the existing result-download race or promise evidence survives a concurrent checkout reset; that limitation is recorded in the separate follow-up. No new cross-run archive or retention daemon is needed. +Write sidecars under the existing checked `.testshards/results` tree, outside disposable scratch, and download them with the other reports. Keep at most one attempt's bounded resource diagnostics per worker. A failed download leaves that attempt locked; it does not start a growing queue of retained runs on the same worker. The next run's results reset is allowed only after retrieval completes and the lock is released. Coordinator-side `.test-runs/` retention is unchanged. This bounds worker storage without expiring undownloaded evidence. No new cross-run archive or retention daemon is needed. ## Authority and derivation After proposal convergence, derive the smallest changes in the existing owners: - [`testing-policy.md`](../../specs/arch/testing-policy.md): built application as the default for product browser acceptance when development-module behavior is not under test; worker-owned reusable test runtimes and cleanup expectations. -- [`blaxel-testshards.md`](../../specs/arch/test-runner/blaxel-testshards.md): attempt temporary directories, cleanup after process teardown, safe orphan reclamation, and bounded worker-side diagnostic retention. The lease/TTL contract stays intact. +- [`blaxel-testshards.md`](../../specs/arch/test-runner/blaxel-testshards.md): exclusive worker lock through setup, process teardown and result retrieval; no time-based reclamation of uncertain work; recovery and capacity consequences; the actual older-runner behavior; pool-script locking; attempt temporary directories, recurring exact-prefix cleanup and bounded worker retention. Replace its TTL availability escape and shell-exit unlock expectations rather than leaving conflicting requirements. - [`reporting.md`](../../specs/arch/test-runner/reporting.md): minimum resource observations and artifact references, bounded content, unavailable evidence, and retention alongside failed/recovered native attempts. -Use existing lifecycle and reporting contracts in `test-runner.md` and `sharded-execution.md`; add a cross-reference only if derivation reveals an actual ownership gap. No `preflight.md` delta is planned. Do not turn optional investigation instrumentation into requirements spread across five specs. +Use existing lifecycle and reporting contracts in `test-runner.md` and `sharded-execution.md`, adding only the cross-references or ownership corrections required by the outer Blaxel lock. Existing isolated-GitHub-only lifecycle fixtures keep their placement. No `preflight.md` delta is planned. Optional investigation instrumentation remains optional. Update the mirrored proofs after independent spec convergence. Test setup changes may also require proof-only corrections to descriptions of the affected browser walks; they do not change their product promises. Do not prescribe a new framework or adopt unrelated harness internals into spec authority. @@ -170,7 +192,15 @@ Update the mirrored proofs after independent spec convergence. Test setup change All execution is on Blaxel at committed, pushed revisions. The shared development host performs file inspection, edits, Git operations, and lightweight remote coordination only. No tests, builds, browser launches, lint, or type checks have run for this proposal. Apply the test-iteration discipline: red/green at the narrowest relevant file, then affected surfaces, then full verification with the default retry budget. -The proofs should establish a small number of consequential boundaries: reuse and final disposal of real runtime files; partial-build cleanup; preservation of a live job's files while orphaned owned scratch is reclaimed; real environment propagation into Node and Chromium; and retained, bounded failure evidence when an attempt fails or its worker exits. Contract coverage handles unavailable probes, unchanged original outcomes, and retention limits. A live targeted Blaxel run and a live planned shard run establish the provider paths. Lease-redesign experiments and result-race repair tests are outside scope. +Derive proofs for the principal paths without a combinatorial fault matrix: + +- One run excludes a competing coordinator through preparation, teardown, and a deliberately delayed result download. Passing the former TTL never admits a second run while processes or retrieval remain. Exercise both targeted and planned dispatch through the real provider. +- A lost coordinator, ambiguous dispatch, or failed retrieval keeps the lock held. Recovery after the coordinator has ended stops only attributed processes, retrieves the available evidence, and then permits reuse. Contract tests cover error handling; real seam coverage must include dispatch and retrieval, not just mocked API responses. +- The inspected older runner does not reclaim a new non-expiring held lock. Legacy helper residue is removed on the next updated run. Pool-script mutation respects the same lock. An unexplained test process or ownership record prevents cleanup/dispatch rather than authorizing deletion. +- Exact-prefix and stale-scratch cleanup preserves unrelated files and live consumers. Real runtime builds demonstrate once-per-worker reuse, final disposal, and partial-build cleanup. A Blaxel observation verifies Node temporary paths and Chromium shared-memory descriptors under the attempt's `TMPDIR`. +- A failed or recovered acceptance retains bounded start/end/OOM evidence through retrieval, including when its worker exits. Contract coverage handles unavailable probes, unchanged original test outcomes, truncation, and limits. + +The live recovery checks use small, disposable, owned processes on private workers. No deliberate OOM, process storm, or invocation of the repository's isolated-GitHub-only destructive fixtures is authorized. If a required assertion cannot be proven safely within Blaxel-only execution, report that specific blocker at proof review; do not silently substitute local or GitHub execution. Use a dedicated two-worker pool named **`rose-cleanup`**, with workers `tv-testshard-x64-rose-cleanup-01` and `-02`, for cleanup, orphan, and interruption experiments. Explicitly pass `--pool rose-cleanup` through the canonical runner; reduced planned runs also need the documented shard-count override. Exercise interruption with disposable owned subprocesses there, without intentionally exhausting memory or running repository fixtures restricted to isolated GitHub hosts. Never run those experiments on shared `poc` workers. Two workers permit a distinct replacement after a contaminated attempt; they do not guarantee capacity for every retry after repeated contamination, which remains an infrastructure failure with the default retry budget intact. @@ -178,10 +208,12 @@ Re-run the converted acceptances without weakening their user-visible assertions For the final gate, provision a separate sandbox named **`tv-cleanup-coordinator-rose`**, excluded from testshard pool labels. It will fetch the committed, pushed branch and run `npm run verify -- blaxel`, so the invoking machine's lint/type-check/package-manifest phases also execute on Blaxel. The two experimental workers and coordinator are a plan, not currently provisioned capacity. Provision and verify their identities before starting validation; if unavailable, report blocked rather than run anything on this host or silently substitute GitHub execution. The user has already authorized outside-pool Blaxel hosts. -The experiment allocation uses at most three concurrent sandbox slots beyond the shared 72: **72 + 2 + 1 = 75**. Check actual workspace usage before provisioning/dispatch because other work may already consume that spare capacity; do not claim those slots are reserved or alter shared workers to obtain them. Delete this contribution's experiment workers before the final standard 36-shard verify, leaving only its coordinator alongside the shared pool. That final run uses the normal `poc` lease path without intentional contamination or interruption. Archive its reports before disposing the coordinator. Record the exact tested revision, failures, recovered flakes, and any measurement limits. Ready-PR CI remains the repository's merge requirement; the contribution's execution evidence stays on Blaxel as instructed. +The experiment plan adds at most three sandboxes. The spec's pool-size and tier examples are not a current capacity inventory: inspect actual workspace usage and provider limits before provisioning/dispatch. Do not claim slots are reserved or alter shared workers to obtain them. Delete this contribution's experiment workers through the pool script after retrieving their reports and before final standard 36-shard verify, leaving only its coordinator. That final run uses the normal `poc` path without intentional contamination or interruption, after old in-flight runs have finished and active coordinators carry the lock correction. Archive its reports before disposing the coordinator. Record the exact tested revision, failures, recovered flakes, and any measurement limits. Ready-PR CI remains the repository's merge requirement; the contribution's execution evidence stays on Blaxel as instructed. Keep browser-version alignment, shard count, browser request limits, retries, and production memory behavior outside this contribution. Forced GC is an investigation tool, not a proposed mitigation. The demo-mode and drag flakes have not been proven to share the resource-exhaustion cause. ## Review status -Proposal round 1 at `1652787e`: FAIL on lease scope and mixed-version consequences. The [review response](test-harness-resource-cleanup-review-response.md) records dispositions. Revised proposal: awaiting independent re-review. No spec, proof, test, or implementation edits yet. Josh's shared-pool rollout option/timing remains an explicit decision, separate from approval of the code contribution; his spec acceptance remains due at PR time. +Proposal round 1 at `1652787e`: FAIL on lease scope and mixed-version consequences. The response at `99b7ed93` narrowed the proposal while those human scope decisions were pending. Josh's 2026-10-27 instruction brings lock correction and recurring legacy cleanup into this PR and removes the pool-recreation choice. The [review response](test-harness-resource-cleanup-review-response.md) records the current dispositions without changing the independent review's historical verdict. + +Revised proposal: awaiting independent re-review. No spec, proof, test, or implementation edits yet. No tests, builds, remote experiments, or worker operations were performed for this revision. Unknowns are live provider behavior during failed dispatch/recovery, actual `TMPDIR` placement and post-mitigation memory savings, current capacity, and the availability impact of locks retained after failures. Josh's calibration limits lock work to our runner and pool scripts. His spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index d95068f1..277a25fb 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -1,28 +1,40 @@ -# Resource-cleanup proposal: round 1 response +# Resource-cleanup proposal: review response -Reviewed proposal: `1652787e`. Independent reviewer: Claude Fable (Nina), arranged by the supervisor. [Full review](test-harness-resource-cleanup-review-r1.md): FAIL on two blockers. This response changes proposal documents only; no spec, proof, or implementation work has started. +Independent round 1 reviewed `1652787e`. Reviewer: Claude Fable (Nina), arranged by the supervisor. The [full review](test-harness-resource-cleanup-review-r1.md) remains unchanged: FAIL on two blockers. The response committed at `99b7ed93` narrowed the work pending Josh's decisions. This response records the revised proposal following his 2026-10-27 instruction; it does not claim reviewer convergence. + +## Josh's scope decisions + +All of the following belong in PR #12: + +- Keep a worker exclusively reserved while the previous run's processes may be alive and until results have been downloaded. Account for direct Blaxel API callers as well as shard coordinators. +- After taking that lock, every run removes the known `television-source-cli-e2e-*` and `television-web-bundle-*` directories directly under `/tmp`, plus stale per-run scratch. Preserve everything else. This handles residue from test revisions without the helper fixes. +- Keep worker-lifetime helper cleanup, per-run scratch, suitable built-app acceptances, and bounded failure diagnostics. Drop pool drain/recreation and the pending rollout choice. + +Josh then calibrated the lock scope: only our runner and pool scripts touch these workers. Keep their locking correct and simple; do not defend against hypothetical callers or add elaborate mixed-version machinery. The revised proposal uses the existing lock directory and ownership identity, holds it through teardown/download, and makes pool operations respect it. It does not introduce a persistent compatibility fence, separate worker controller, or general API gateway. + +The [proposal](test-harness-resource-cleanup-proposal.md) incorporates the result-download-race note. Its separate file is removed because the correction is now in scope. No spec, proof, or implementation has been changed. ## Blockers -| Finding | Assessment and disposition | +| Finding | Current assessment and disposition | | --- | --- | -| B1: lease redesign is unnecessary for safe scratch cleanup and leaves availability/mixed-version consequences unresolved | Accepted. Remove the lease redesign, indefinite unavailability rule, and report-root migration. Keep the current TTL and release behavior. Private attempt directories and process-identity checks govern disposal independently of leases. Unknown ownership means retain that directory, not permanently quarantine the worker. Record the real result-download race as a separate follow-up, including the current timeout/TTL bounds and consequences a future proposal must address. No decision to replace TTL is being made for Josh. | -| B2: old branches continue leaking outside the new directory | Accepted. Restrict cleanup claims to runs carrying the change. Explicitly describe continuing old-branch residue and why an initial cleanup may need repetition. Recommend tolerating residue during migration, then a coordinated drained-pool recreation using existing provisioning. Present alternatives and leave the shared-pool option and timing to Josh. That operation is neither performed nor silently authorized by accepting the code proposal. | +| B1: lease redesign is unnecessary for scratch cleanup alone, with unresolved availability and mixed-version consequences | The original scope objection was warranted. Josh now explicitly requires a simple correction to our lock lifecycle. Keep atomic directory acquisition and `lockId`; remove age-only reclamation, audit descendants, and release from the coordinator after downloads. Cover dispatch uncertainty and signal/exception paths using the existing cleanup machinery. State the availability cost: uncertainty can hold a worker indefinitely. Keep the fixed results tree and protect it through download instead of adding a cross-run artifact archive. | +| B2: old branches keep leaking outside the new directory | Address this with Josh's selected recurring pre-run deletion. New locks use `expiresAt: 0`, which the inspected old acquisition code does not reclaim. Older coordinators still release their own locks before downloading, so active dispatching branches need the small lifecycle correction and old in-flight downloads must finish. No permanent old-client exclusion, separate controller architecture, or pool recreation is proposed. Helpers on older branches can still leak; subsequent updated runs reclaim their known residue. | ## Refinements -| Finding | Assessment and disposition | +| Finding | Current disposition | | --- | --- | -| R1: quantify relative benefit | Accepted. Add the roughly 3.2 GB test-induced shared-memory increase and low headroom from the passing experiment, versus 1.16 GB / 0.079 GB of the two leaked prefixes on inspected workers. Keep the expected saving from built loading distinct from a measured post-fix saving. Intentional checkout/browser/cache occupancy remains outside cleanup. | -| R2: compare pool recreation with legacy deletion | Accepted. Prefer existing drain/delete/ensure over bespoke legacy-deletion code. The alternative prefix removal would be an operator procedure on explicitly drained workers, not routine pre-run behavior. Josh owns the shared-pool disruption and timing. | -| R3: commit only to minimum diagnostics | Accepted. Require parent start/end RAM, shared-memory, temporary-free-space and OOM-counter observations plus bounded kernel OOM lines on failure, linked to existing native attempt reports. Retain failures recovered by retry. Continuous sampling, descriptor inventories and other detailed probes are optional investigation tools. Fixed-size snapshots and a bounded excerpt satisfy retention without a monitoring framework. Reduce derivation to the relevant existing spec owners. | -| R4: name validation infrastructure and account for capacity | Accepted. Plan two private `rose-cleanup` workers for cleanup/interruption experiments and `tv-cleanup-coordinator-rose` for the final gate's invoking-machine phases. They are not yet provisioned or reserved. At most three additional slots fit the documented 72+3=75 bound only if no other work consumes that headroom; check usage first. Remove the private workers before a standard final verify on the shared pool. No fallback to this host or substitution of GitHub execution: the user's Blaxel-only instruction takes precedence over the suggested CI alternative. | -| R5: disclose remaining unbundled exposure | Accepted. Add a named inventory and concrete reload/multiple-page cases: telemetry, navigation setup, authentication reconnect, and Electron theme resets. Source inspection found 38 test files with a literal app-entry reference plus two indirect helper consumers; 41 includes their shared helper. This minor counting distinction does not change the finding: roughly two dozen test files remain exposed. Explain the in-process server and Electron boundaries without declaring them inherently dependent on Vite or claiming adequate headroom. | -| R6: justify helper fixes in addition to scratch | Accepted. Helper teardown frees within-shard worker outputs and protects runners with no Blaxel wrapper. Attribute this host's directory counts to the independent review; no local cleanup is part of this work. | -| R7: sequence PR #9 | Accepted with current-head qualification. Recommend #9 lands first, then integrate its final changes before affected-file validation and ready review. Keep its theme-reset ownership. The inspected #9 head `4e85e733` changes theme/auth files but does not list `stable-front-proxy.ts`; do not assume an unmerged proxy hook exists. Reconcile the final helper API if it changes. The supervisor coordinates merge order. | +| R1: quantify relative benefit | Retain the historical roughly 3.2 GB shared-memory increase and 1.16 GB / 0.079 GB leaked-prefix measurements. Explicitly assess PR #32: it directly addresses 304-buffer accumulation for registry Vite services. Keep suitable built-app acceptances for shipped-asset coverage and fewer module requests, with no claim that they are necessary to fix that leak or will save the historical amount of memory. Any incremental resource measurement uses a baseline containing #32. | +| R2: compare pool recreation with legacy deletion | Josh chose recurring exact-prefix cleanup. Remove the alternatives table, recreation recommendation, and unresolved rollout decision. Require whole-worker exclusion and idle confirmation before deletion; the prefix alone is insufficient evidence of disuse. Preserve unrelated `/tmp` contents and persistent caches. | +| R3: commit only to minimum diagnostics | Keep parent start/end RAM, shared memory, temporary free space and OOM-counter observations, plus bounded kernel OOM lines on failure. Retain evidence for recovered retries. Keep the 128 KiB/surface and 8 MiB/attempt limits. Hold one attempt's evidence on a worker until retrieval; a blocked worker cannot accumulate new attempts. Detailed probes remain optional. | +| R4: name infrastructure and account for capacity | Retain two private `rose-cleanup` workers and the separate Blaxel coordinator as planned infrastructure, not reserved capacity. Recheck actual pool inventory and provider limits rather than assuming the old 72+3 arithmetic describes today's workspace. All test/build execution remains on Blaxel; private experiments use bounded owned processes and do not invoke isolated-GitHub-only destructive fixtures. | +| R5: disclose remaining unbundled exposure | Retain the named inventory, labeled with its original base revision, and require reconciliation against current main before implementation. Existing deferred cases remain visible; no claim that all of them require Vite or have sufficient headroom. | +| R6: justify helper fixes in addition to scratch | Keep helper cleanup for within-shard worker disposal and callers without a Blaxel wrapper. Preserve once-per-worker/per-version build reuse and clean failed builds. This contribution does not authorize cleanup on the shared development host. | +| R7: sequence PR #9 | PR #9 merged as `f81a2465`. Preserve its theme-reset conversion when integrating current main and reconcile the other theme/auth walks and stable-front proxy against the merged implementation. There is no remaining PR #9 merge-order decision. | -## Decision still needed from Josh +## Remaining evidence -Choose the shared-pool migration option and timing. Recommendation: tolerate old-branch residue temporarily, have active branches take the cleanup change, then drain and recreate the pool. This buys simplicity at the cost of temporary reduced headroom, a coordinated interruption, and cold caches. An earlier recreation may be justified by capacity but can be contaminated again by old branches. Proposal convergence and code work can proceed without performing this operation; pool-wide cleanliness cannot be claimed until it happens and leaking old runs stop. +Independent re-review should assess the revised scope against Josh's decisions and simplicity calibration: our acquire/release/error paths, result retrieval before release, recovery of uncertain work, and the pool-management script. There is no adversarial API-caller requirement. Live provider behavior and `TMPDIR` placement still need proof-stage experiments. No observed Chromium failure has been attributed to the lock or download races, and no post-#32 experiment establishes the remaining memory benefit of built loading. -Validation of this revision consists of read-only code/policy/PR inspection and document consistency/whitespace checks. No tests, builds, browsers, or servers were run on this host, and no remote experiments or cleanup were performed during remediation. The supervisor will receive the pushed revision for independent re-review. +Validation of this revision consists of document, code, policy, and PR inspection plus Git whitespace/diff checks. No tests, builds, browsers, servers, remote experiments, or worker cleanup were run for this revision. The pushed proposal is the next independent-review checkpoint; spec derivation follows convergence. From 57a701d067bfd1333d091a8aa04716c69744d121 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:05:38 +0000 Subject: [PATCH 04/64] docs: define bounded Blaxel worker recovery Specify coordinator renewal, owned-process cleanup before reuse, and the disposition of abandoned reports. Narrow built-app conversion to two files after the Vite response fix, and address proposal review refinements on scratch cleanup, old branches, browser evidence, and remote credentials. --- .../test-harness-resource-cleanup-proposal.md | 107 +++++++++++------- ...test-harness-resource-cleanup-review-r2.md | 72 ++++++++++++ ...arness-resource-cleanup-review-response.md | 38 ++++++- 3 files changed, 173 insertions(+), 44 deletions(-) create mode 100644 docs/working/test-harness-resource-cleanup-review-r2.md diff --git a/docs/working/test-harness-resource-cleanup-proposal.md b/docs/working/test-harness-resource-cleanup-proposal.md index 7df17f63..abf3839b 100644 --- a/docs/working/test-harness-resource-cleanup-proposal.md +++ b/docs/working/test-harness-resource-cleanup-proposal.md @@ -8,7 +8,7 @@ In the investigated runs, repeated unbundled application loads consumed several The fixes have different effects. The historical theme-reset walk added roughly 3.2 GB of shared memory in a passing run that reached about 0.55 GB available RAM. Built loading removes hundreds of development-module requests per load, but those historical numbers cannot predict its incremental saving after the Vite response correction described below. By comparison, the two leaked runtime prefixes occupied 1.16 GB on the worst worker inspected and 0.079 GB on the other. All of `/tmp` occupied 1.20 GB and 0.16 GB respectively. Cleanup recovers accumulated capacity; it does not guarantee browser headroom. The original worker also retained roughly 1.4 GiB of checkout/dependencies, 0.9 GiB of browser installations, and other intentional caches. Those are outside this cleanup. -This contribution fixes the cached runtime lifetimes, runs suitable product acceptances against the built application, and retains compact resource evidence when an acceptance fails. It also makes the worker reservation cover preparation, execution, process teardown, and result download. After taking that lock and confirming the worker is idle, each run removes the two known leaked-runtime prefixes and stale run scratch. This repeatedly reclaims residue from older test revisions as well as newer ones. Cleanup preserves active jobs, provider services, deliberate dependency caches, and undownloaded reports. +This contribution fixes the cached runtime lifetimes, moves five acceptance cases in two files onto the built application, and retains compact resource evidence when an acceptance fails. It also makes the worker reservation cover preparation, execution, process teardown, and result download, with automatic recovery when a coordinator stops renewing its reservation. After taking that lock and confirming the worker is idle, each run removes the two known leaked-runtime prefixes and stale run scratch. This repeatedly reclaims residue from older test revisions as well as newer ones. Cleanup preserves provider services, deliberate caches, and the current run; abandoned runs and reports have the bounded recovery rule below. PR #9 merged as `f81a2465` on 2026-10-01 and owns the built-application conversion of **“executable theme reset reloads connected Chromium documents under confirmed state.”** Reuse that change when integrating main; do not duplicate it. Main also includes `982e564d` (PR #32), making registry Vite services answer conditional requests in full. Its runner spec records Chromium 145 retaining about 2 MB per 304 response, roughly 465 MB per application reload. That newer mitigation changes the baseline for future measurements. The investigation numbers above describe the historical runs, not current main's residual memory cost; neither the helper leak nor the lock/download gaps is repaired by the Vite response change. @@ -35,6 +35,12 @@ There are two proven helper leaks: Other `/tmp` entries include Playwright artifacts and transform caches, Node's compile cache, `tsx` caches, Chromium temporary files, X11 state, an `xvfb-run` authority directory, the Git askpass helper, hundreds of uploaded shard scripts, test-runner fixtures, and `multipart-uploads`. The idle process tables include Blaxel's `sandbox-api`, `ukp-fs`, and entrypoint processes. The precise owner of every miscellaneous entry was not established; in particular, the upload directory must be treated as outside Television's disposal ownership. Neither idle snapshot had an open `/tmp` descriptor or cwd reference, which establishes only those instants. +### Browser versions in the evidence + +The supplied failure trace records **Chromium `148.0.7778.96`**. During the investigation, the coordinator made cached build 1223 available at the build-1208 path expected by Playwright 1.58.2, whose manifest names Chromium `145.0.7632.6`. PR #32's separate 304-response evidence concerns Chromium 145. These are distinct observations; the 145 finding does not establish the installed version or exact retention behavior on every Blaxel worker. + +**The version workers run today is unknown.** At inspected main `a8e73fbf`, the coordinator still creates those aliases when build 1223 exists and the expected build-1208 path does not. That conditional code cannot establish the current contents of each persistent worker's browser cache. During the planned Blaxel validation, record the actual launched browser version and resolved executable/cache path with the run evidence before interpreting resource measurements or `TMPDIR` observations. Do not infer the running browser from the Playwright package version. Browser-version alignment remains outside this contribution. + ## Worker exclusion, result retrieval, and safe `/tmp` cleanup Josh has selected routine cleanup under an exclusive worker lock. There is no pool drain-and-recreate plan or outstanding rollout choice. Before each run, remove only the two known legacy prefixes and stale scratch in the dedicated runner namespace; after execution, remove that attempt's scratch only after its processes stop and its evidence has been finalized outside scratch. Never delete all of `/tmp` or use age alone as proof of disuse. @@ -47,35 +53,56 @@ The canonical coordinator intends one shard job per sandbox. [`acquireAny()` and 2. The shard shell's `EXIT` trap releases the lock before `runShard()` downloads results from `/workspace/television/.testshards/results`. A following checkout clears that fixed tree. This leaves an unprotected interval in which the previous job's evidence can disappear or be replaced. Report identity checks can detect some mismatches but cannot restore deleted files. This source-level race is present at both the branch base and inspected main; no collision was observed during the Chromium investigation. 3. Direct Blaxel process API calls bypass this repository lock. The inventory used that API. `manage-blaxel-testshards.mjs` also provisions and checks workers through `process.exec`, and deletes them through the provider API. The idle snapshots establish neither provider serialization nor a history-wide absence of overlap. -These are demonstrated protocol gaps, not demonstrated causes of the Chromium failures. The download-race follow-up is incorporated here; both its correction and the removal of time-based reclamation belong in PR #12. +These are demonstrated protocol gaps, not demonstrated causes of the Chromium failures. The download-race follow-up is incorporated here. Replace blind TTL reclamation with a renewable reservation whose expiry requires process cleanup before reuse. ### One lock through process teardown and download Josh's calibration is that only our runner and pool scripts touch these workers. Correct those code paths using the existing lock; do not add a worker controller, persistent compatibility fence, access-control system, or defenses against hypothetical API callers. -Retain the worker lock until both conditions hold: **all execution for the attempt is stopped, and its results are downloaded and validated at the coordinator.** Keep atomic directory acquisition and the existing `lockId` ownership check. Write new held locks with `expiresAt: 0`: the inspected older runner reclaims only a positive timestamp in the past, so it also leaves these locks alone. Remove the new runner's age-only reclamation. Timeouts still bound execution and trigger cleanup, but they never authorize another run to overwrite a live attempt or its undownloaded results. The directory is removed on confirmed release, as it is today; it does not become a permanent worker installation. +For a participating coordinator, retain the worker lock until both conditions hold: **all execution for the attempt is stopped, and its results are downloaded and validated at the coordinator.** Keep atomic directory acquisition and the existing `lockId` ownership check. Write `expiresAt: 0` for compatibility: the inspected older runner reclaims only a positive timestamp in the past, so it leaves these locks alone. The updated runner uses a separate renewable recovery deadline, described below. That deadline authorizes cleanup of an abandoned attempt, never immediate reuse beneath surviving processes. Normal release removes the directory, as it does today. The runner's order becomes: -1. Acquire the lock, then confirm the worker has no prior test/shard work before removing known leftovers. Extend the existing owner-token attribution to the outer shard, including checkout and setup children. Record the remote process name and PID/start identity so recovery can distinguish the run from PID reuse; the coordinator's host/PID alone is insufficient. -2. Prepare and execute the shard under that lock. Keep the existing all-shards-acquired barrier, sequential surfaces, and attempt/retry behavior. Record that dispatch was requested before making the API call. If its outcome is unknown, retain the lock until execution status and cleanup are confirmed. +1. Acquire the lock, recovering an abandoned attempt if necessary, then confirm the worker has no prior test/shard work before removing known leftovers. Extend owner-token attribution to the outer shard, including checkout and setup children. Preserve that attempt token alongside surface tokens, which nested supervisors replace today, so detached descendants remain attributable to the attempt. Record the remote process name and PID/start identity so recovery can distinguish the run from PID reuse; the coordinator's host/PID alone is insufficient. +2. Renew the reservation while preparing, executing, cleaning up, and downloading. Keep renewal active while waiting at the existing all-shards-acquired barrier. Keep sequential surfaces and attempt/retry behavior. Record that dispatch was requested before making the API call. If its outcome is unknown, retain ownership while resolving it; a coordinator that exits leaves the bounded recovery path to the next acquirer. 3. On success, failure, timeout, or interruption, stop and audit the run's attributed processes, including descendants of an exited shell. Reuse the existing owner audit and TERM/KILL cleanup. A shell exit or successful kill API response alone is insufficient. The shell finalizes reports but no longer releases the lock in its `EXIT` trap. 4. Download the existing summary, native reports, logs, setup/cleanup evidence, and new diagnostic sidecars to the coordinator's run directory. Validate the existing run/attempt identities and required files. Missing or corrupt evidence remains an infrastructure failure; an early setup failure records which reports were never produced and retrieves what is available. 5. Release the matching lock only after teardown and retrieval complete. Keep the fixed checked results tree: protecting it through download closes the checkout-reset race without a separate artifact archive. Release unused locks after acquisition/upload failures only when execution was never requested and there are no pending writes or produced results to retain. -All runner exit paths must follow that order, including acquisition rollback, architecture rejection, both signal paths, retry, and exception cleanup. An interrupted download retains the lock; retry retrieval without repeating the tests. A failed teardown or uncertain dispatch excludes that worker from retries. Keep the default retry budget. +All runner exit paths must follow that order, including acquisition rollback, architecture rejection, both signal paths, retry, and exception cleanup. An interrupted download retains the lock; retry retrieval without repeating the tests. If the coordinator exits before completing that work, it stops renewing and reports the reservation's recovery deadline. A failed teardown or uncertain dispatch excludes that worker from immediate retries. Keep the default retry budget. + +### Automatic recovery with one renewable deadline + +**A coordinator keeps its reservation by renewing it; after renewal stops for the recovery interval, the next acquirer stops and audits that run's owned processes, clears its abandoned state, and takes the lock.** The same procedure handles an already quiet worker and a surviving shard supervisor or test descendant. An expired deadline alone never makes the worker available. + +Use the coordinator to renew once a minute, including during result download. Each successful renewal records worker-local time and sets `recoverAfter` to that time plus **the attempt's configured worker timeout plus 15 minutes**. The cold-worker default of 480 seconds therefore permits recovery 23 minutes after the last renewal; 240 seconds gives 19 minutes. A longer configured timeout gets the same margin. Record that timeout when acquiring the lock, so custom runs and time spent before dispatch are covered. Renewal requires the same owner and a still-valid deadline; it cannot revive an expired reservation. Only the coordinator renews: a surviving supervisor cannot preserve a crashed coordinator's reservation forever. No daemon, cross-host PID probe, or separate heartbeat service is needed. + +The automatic entry point is the canonical runner's `acquireAny()` / lock helper. On finding an expired `recoverAfter`, it performs this bounded worker-local sequence before publishing a new owner: -### Recovery, old branches, and pool operations +1. Re-read the matching lock identity and deadline under a short filesystem critical section shared by acquisition, renewal, dispatch registration, and release. Use an OS-released advisory lock (for example `flock` on a fixed guard file outside the disposable lock directory), so a killed recovery process cannot leave a second permanent mutex. Hold it only for worker-local state changes and bounded cleanup, not network downloads. A competing acquirer or late renewal cannot act on a stale read. +2. Stop only the old run's attributed supervisor and descendants, using its owner token and the existing PID/start-identity checks, TERM/KILL escalation, and owner audit. The current stale-owner reaper deliberately leaves live supervisors' children alone; recovery must stop the abandoned run's live supervisors as well, without weakening the ordinary reaper's protection for other owners. Confirm no live or stale owners or remaining attributed processes before deleting their files. A dead supervisor is not enough; detached descendants still count. A late remote dispatch must check the matching, unexpired reservation and register ownership under the same critical section **before** checkout/setup can start. This prevents an API request delayed across recovery from launching the old run into the new checkout. Uploads use attempt-specific scratch paths. +3. Remove the abandoned attempt's scratch and the allowed legacy leftovers, then reset its results as part of the next run's normal preparation. **Discard undownloaded results of this abandoned attempt after the recovery interval.** Do not create a second archive or require an orphan-report download that can itself block reuse. Record the prior worker/run/lock, last renewal and deadline, stopped process identities, cleanup outcome, and whether reports were discarded in a bounded recovery receipt with the incoming attempt's setup sidecars, outside the old results tree being reset. Loss of reports never becomes a passing result for the abandoned run. +4. Publish the incoming reservation only after cleanup succeeds. Recovery is repeatable: if its process dies midway, the old expired identity remains until another acquirer finishes the same work. Normal release and renewal check `lockId`; a returning coordinator that has lost it must stop work and report infrastructure failure, never release or mutate the new owner's attempt. -A coordinator crash can leave a lock indefinitely. Report the worker, lock/run identity, last phase, and paths needed for recovery. Once the original coordinator is confirmed ended, recovery reuses the same stop/audit, download, and matching-lock release steps. If execution status, process teardown, or retrieval cannot be confirmed, leave the lock held and report the specific reason. No timer or blind `rm -rf` is a recovery procedure. This availability cost is explicit; no heartbeat service or general ownership-transfer protocol is proposed. +This defines coordinator loss operationally: it has stopped renewing for longer than a complete worker timeout plus a generous margin. It does not prove that a remote OS process died. A network partition or suspended coordinator beyond that interval has the same consequence as a crash. A healthy coordinator keeps renewing through a long download, so elapsed shard time alone does not end retrieval protection. Once renewal is lost, orphaned reports are the explicit exception to normal download-before-release. Their file timestamps do not extend the deadline; a hung child must not defeat recovery by continuing to write. -Older branches matter in two concrete ways. Their helpers still leave the two known directory prefixes, which every updated run removes. Their coordinators also unlock at shell exit and use timestamp expiry. New non-expiring locks stop the inspected old acquisition code from taking an updated run's worker, but cannot correct an old coordinator's own early release. Carry the small lock lifecycle correction into branches still dispatching to the shared pool, and let already-started old runs finish their downloads before relying on the corrected behavior. There is no permanent old-client exclusion mechanism or separate controller/test-revision architecture. An existing legacy lock, even expired, is not reclaimed merely because of its age; uncertain ownership needs the same recovery checks. No pool drain/recreation is required to reclaim test residue. +The capacity cost is bounded by the remaining recovery interval, the next acquisition attempt, and bounded process cleanup. Quiet crash leftovers wait the same interval as active ones; accept that small availability cost to avoid a second liveness test, report-age clock, or special quiet-run recovery procedure. The 15-minute margin is a proposed operating allowance, not a measured provider guarantee. Verify renewal and cleanup timing on private workers before relying on it. -`manage-blaxel-testshards.mjs` is the other actual API caller to correct. Its mutation commands must respect the same lock: obtain locks on the affected existing workers before deletion and refuse busy workers, releasing unused locks if acquisition fails. Fresh provisioning runs with dispatch to that pool stopped until setup and architecture verification finish. Listing/status is read-only. This covers our pool script without adding a generic API gateway or maintenance framework; manual pool operations use the same drained-pool discipline. +Two correctness limits remain. Failure to identify or stop a process must hold and report the worker rather than run tests over it; an unkillable process or broken provider needs operator repair, not another expiry that ignores the failed audit. Missing or corrupt **lock ownership** is also different from missing scratch metadata: dispatch cannot precede complete ownership registration, and an empty acquisition left before dispatch can be reclaimed after a clean worker audit, but ambiguous live work cannot be killed by a name guess. These limits prevent unsafe reuse; an ordinary crashed coordinator with attributable surviving processes follows automatic recovery and does not need an operator. + +### Older branches and pool operations + +Older branches matter in two concrete ways. Their helpers still leave the two known directory prefixes, which every updated run removes. Their coordinators also unlock at shell exit and use timestamp expiry. New locks' `expiresAt: 0` stops the inspected old acquisition code from taking an updated run's worker, but cannot correct an old coordinator's own early release. Branches take the correction when they integrate main; updating every active branch or waiting for all old runs is not a prerequisite for this contribution or its final verification. + +A legacy lock has no renewable deadline or complete outer-run attribution. Respect its unexpired TTL. After expiry, reclaim it only following a clean worker audit; keep and report one with live or ambiguous work that cannot be attributed safely. Do not infer an owner token from a process name or pretend an old coordinator supplies new renewal evidence. This limited legacy exception can cost capacity, but it avoids new mixed-version machinery. No permanent old-client exclusion mechanism or pool recreation is proposed. + +`manage-blaxel-testshards.mjs` is the other actual API caller to correct. Its mutation commands must respect the same reservation: obtain locks on affected existing workers before deletion and refuse busy workers, releasing unused locks if acquisition fails. Reuse the common acquisition/recovery helper rather than inventing another expiry rule. Fresh provisioning runs with dispatch to that pool stopped until setup and architecture verification finish. Listing/status is read-only. This covers our pool script without adding a generic API gateway or maintenance framework; manual pool operations use the same drained-pool discipline. + +Preserve the runner's read-only `--audit-stale-owners` mode: it must not recover another run or remove residue before observing it. It may reserve an available worker without recovery, holding that reservation through its existing dependency setup and report collection. A held lock is reported/skipped. This exception preserves the diagnostic's purpose; ordinary test acquisition and pool mutations use automatic recovery. ### A directory for each shard attempt -Give every locked shard **attempt** a fresh, short, private directory beneath a runner-owned root, for example `/tmp/tv-testshards//`. Record its run, shard, attempt, lock identity, and remote supervisor identity outside its disposable `tmp/` child. Record ownership before using scratch; a pre-dispatch directory must not be mistaken for an abandoned run. Upload/dispatch failure removes only this attempt's directory when execution is known not to have started and pending requests have settled. An unknown outcome leaves the lock and scratch for recovery. Use a short path because browser and Electron processes can create Unix sockets with limited pathname length. +Give every locked shard **attempt** a fresh, short, private directory beneath a runner-owned root, for example `/tmp/tv-testshards//`. Record its run, shard, attempt, lock identity, and remote supervisor identity outside its disposable `tmp/` child for diagnostics. The current attempt's directory is explicitly excluded from pre-run cleanup, including before dispatch. Missing or incomplete metadata in an earlier attempt's scratch does not add a cleanup gate once the lock and worker audit establish disuse. Upload/dispatch failure removes only this attempt's directory when execution is known not to have started and pending requests have settled. An unknown outcome leaves the lock and scratch for recovery. Use a short path because browser and Electron processes can create Unix sockets with limited pathname length. Export an absolute `TMPDIR` pointing to that child before running preparation and test commands. Keep scripts, plans, askpass, and setup sidecars within the attempt directory as well. Preserve `TMPDIR` through child environments. Continue to keep deliberately persistent npm, Electron, browser-download, and dependency-hash caches at their existing locations outside scratch space. Playwright's transform cache may become attempt-local initially; measure any resulting setup cost before deciding it deserves a separately bounded persistent cache. @@ -92,13 +119,13 @@ This establishes the intended wiring; a live Blaxel test must still observe Node Before preparation for **every run**, while holding the worker reservation and after confirming no prior execution or retrieval remains, remove: - Real directories directly under `/tmp` whose basenames match `television-source-cli-e2e-*` or `television-web-bundle-*`. The helper source and inventories establish these as leaked test builds; Josh has authorized their recurring removal. Missing per-directory metadata on these legacy copies is expected. The whole-worker exclusion and idle check supply the disuse evidence that the prefix alone cannot supply. -- Stale attempt directories beneath the dedicated `/tmp/tv-testshards/` root, identified by runner metadata and confirmed completed/recovered ownership. Exclude the current reservation's staging directory. A dead supervisor alone does not prove that descendants or delayed dispatch are gone. Missing or inconsistent metadata needs recovery, not an age-based guess. +- Every attempt directory directly beneath the dedicated `/tmp/tv-testshards/` root except the current one, after the worker owner audit confirms no prior live or stale owners remain. These directories belong to our runner. As with the legacy prefixes, the held lock and clean worker audit establish disuse; missing, half-written, or inconsistent per-directory metadata does not prevent removal. This does not reclaim another run's held lock or decide how dead-coordinator locks are recovered. -Revalidate directory identity immediately before deletion; reject symlink roots, unexpected mount boundaries, or paths escaping the exact allowed parent. Recursive removal must not follow nested links to another tree. Preserve the lock directory, provider uploads, X11 state, unrelated temporary files, checkouts, deliberate caches, and every undownloaded report. Do not broaden the legacy patterns to other Television-looking names. Report per-prefix counts, allocated bytes removed, filesystem free space before/after, and skips or errors; those two space measurements need not be equal when files remain open or the filesystem accounts storage differently. +Revalidate directory identity immediately before deletion; reject symlink roots, unexpected mount boundaries, or paths escaping the exact allowed parent. Recursive removal must not follow nested links to another tree. Preserve the lock and guard, provider uploads, X11 state, unrelated temporary files, checkouts, and deliberate caches. Reports stay outside scratch; only abandoned-attempt recovery can discard them before download. Do not broaden the legacy patterns to other Television-looking names. Report per-prefix counts, allocated bytes removed, filesystem free space before/after, and skips or errors; those two space measurements need not be equal when files remain open or the filesystem accounts storage differently. After execution, finalize evidence outside scratch, stop all attributed processes, then remove this attempt's scratch while the reservation remains held through download. Setup failure, acceptance failure, timeout, and interruption use the same ownership rule. If teardown is uncertain, preserve scratch and keep the worker reserved. If deletion fails after confirmed teardown, preserve the original test outcome and add cleanup failure evidence; the worker cannot start another test until pre-run cleanup succeeds. Evidence must still be downloaded. A partial deletion must never be reported as reclaimed capacity for the whole directory. -Unknown ownership or a live consumer means skip deletion **and exclude that worker from test dispatch**, with a recoverable infrastructure failure. There is no promise that every abandoned byte is automatically reclaimable, nor a fallback to deleting unrelated files to obtain headroom. Known residue left by old test revisions is removed on subsequent updated runs without recreating workers or losing their caches. This proposal revision performs no worker cleanup or configuration changes. +An inconclusive worker audit or a live consumer means skip deletion **and exclude that worker from test dispatch**, with a recoverable infrastructure failure. Missing scratch metadata alone does not make that audit inconclusive. There is no fallback to deleting unrelated files to obtain headroom. Known residue left by old test revisions is removed on subsequent updated runs without recreating workers or losing their caches. This proposal revision performs no worker cleanup or configuration changes. ## Worker lifetime for cached runtimes @@ -114,40 +141,38 @@ The helper fix matters even with shard scratch: it releases files between test w ### Assessment after PR #32 -**Keep the conversion set below, for product fidelity and fewer application-module requests; do not present it as necessary to repair the measured 304-buffer leak.** PR #32 (`982e564d`, merged 2026-10-09) removes conditional request headers at the registry Vite service boundary, so those services return full responses. Its recorded Chromium 145 evidence identifies the 2 MB buffer retained per 304 until context closure. That change directly addresses the repeated-reload mechanism for tests using those services, including tests that remain on Vite. Merely counting unbundled modules no longer establishes that the same accumulation persists on current main. +**Convert five cases in two files, rather than the earlier 17-file set.** Josh delegated the scope judgment to this contribution. PR #32 (`982e564d`, merged 2026-10-09) removes conditional request headers at the registry Vite service boundary, so those services return full responses. Its recorded Chromium 145 evidence identifies the 2 MB buffer retained per 304 until context closure. That change directly addresses the repeated-reload mechanism for tests using those services, including tests that remain on Vite. Routine cleanup also recovers the leaked runtime capacity on each updated run. Neither mechanism now justifies a broad application-loader migration. -The listed acceptances assert shipped product behavior and already use a real `ProductServer` with built assets available. Loading those assets exercises the packaged application, asset routing, and authentication/reconnect behavior users receive; a development-module page can miss packaging and delivery mistakes. It also avoids hundreds of module requests per application load, including in multi-page/restart walks. Fewer requests is a concrete change, but its incremental memory and time benefit after #32 is unmeasured. Do not claim a specific reduction, that #32 is insufficient, or that all remaining Vite tests are resource-unsafe. +The two remaining source-loaded theme lifecycle cases are worth converting because they already claim built-application behavior in parts of their proofs and exercise repeated loads and server restarts. Browser demo mode is a small adjacent conversion: its three cases share one product-page helper, with repeated application reloads in two cases. Both files already have a real `ProductServer` and built assets available. This improves their shipped-asset boundary and removes hundreds of module requests per load without a new browser harness. The incremental memory and time benefit after #32 is unmeasured; the demo-mode flake is not proven to have been an OOM. -The conversion cost stays limited to this existing product-server set and the stable-origin restart helper. Preserve the assertions and #9's merged theme-reset conversion. Keep #32 for the remaining Vite services and keep development loading for module/fixture seams. Converting the in-process-server and Electron suites would require broader harness work and is still deferred. Validate the converted paths functionally and confirm their request inventory; any before/after resource comparison must use a baseline containing #32. This is a recommendation based on the tests' intended boundary and source inspection, not a post-#32 memory experiment. +Diminishing returns set in after these two files. The other product walks could gain packaged-asset coverage, but reviewing their loader, authentication, restart, and proof changes adds regression work without evidence of a remaining resource failure that those conversions would cure. Keep #32 for every remaining registry Vite service. Preserve the assertions and #9's merged theme-reset conversion. Validate the selected paths functionally and confirm their request inventory; any before/after resource comparison must use a baseline containing #32. This scope is based on source and proof inspection, not a post-#32 memory experiment. ### Conversion boundaries Use the already-built web application served by `ProductServer`. Existing registry pre-commands build the web and artifact documents before both browser-app surfaces, and `installProductRuntimeAssets()` already copies the application index and assets. Preserve the tests' assertions, real browser behavior, CSS-motion declarations, authentication, and server lifecycle. -The proposed conversion set is the browser product walks using `ProductServer.appURL()`: +The complete conversion set under `packages/web/test/e2e/` is: -| Tests under `packages/web/test/e2e/` | Reason and boundary | +| Tests | Reason and boundary | | --- | --- | -| `channel-product.test.ts`, `channel-crud.test.ts`, `channel-sidebar-remote.test.ts`, `channel-switcher-context-menu.test.ts` | Real channel actions, shared state, drag behavior, and document continuity; no claim about Vite modules. | -| `tab-pages.test.ts`, `keyboard-navigation-product.test.ts`, `artifact-view-wheel.test.ts` | Browser product behavior and real artifact frames. | -| Product-server walks in `top-bar.test.ts` and `auth.01.test.ts` | Built-shell selection, creation, and token behavior. Keep their component fixtures and separate endpoint tests in their current harnesses. | -| `browser-demo-mode.test.ts` | Real server marker and artifact navigation; built loading preserves the behavior being asserted. | -| `markdown-content-sync.spec.ts`, `path-artifact-saga.test.ts`, `path-artifact-real-stack-coverage.test.ts`, `navigable-history-real-stack.01.test.ts`, `navigable-history-real-stack.02.test.ts` | File watching, persistence, editing, and navigation through real product boundaries. A test title saying “source-served” is not itself a requirement for Vite; reconcile its proof citation. | -| `onboarding-browser.test.ts` | Real boot/restart and browser-visible onboarding. Preserve the stable browser origin while the backend binds a new port. | -| The settings-persistence and executable-theme-lifecycle cases in `theme-product-acceptance.test.ts` | Both exercise real server restart, and the executable lifecycle loads two pages repeatedly. Preserve those transitions through a stable origin serving built assets. Leave the theme-reset case to PR #9. | +| `theme-product-acceptance.test.ts`: “theme settings persist rendered app and artifact outcomes through a built-server restart” and “executable themes run only in the Chromium application lifecycle and isolate failures” | Keep the two real documents, executable isolation, appearance, package watching, and server-restart assertions. Current main's [product proof](../../proofs/product/themes-and-appearance.md) calls the executable and appearance walks built-application acceptances, while its selection description still names Vite loading. The [delivery proof](../../proofs/arch/themes/delivery.md) likewise calls the executable frame walk built. Make the setup and proof descriptions agree on shipped assets. PR #9's theme-reset case and the already-built cases are untouched. | +| All three cases in `browser-demo-mode.test.ts` | Change the shared `openApp` helper. The marker-on/off walk reloads twice; return-to-original reloads once. The bridge-inert case shares the same helper and real marker/server boundary, so keeping its loader identical needs no separate harness or additional conversion machinery. Preserve external navigation, unsupported-view behavior, and the inert bridge assertions. | A server URL is sufficient where a backend stays up. Where restart is part of the walk, reuse the existing [`startStableFrontProxy()`](../../test/helpers/stable-front-proxy.ts) to serve the built document and retarget HTTP/WebSocket traffic after restart. Do not weaken a reconnect test by navigating to a replacement origin or clearing browser state. Keep development loading available explicitly for module and fixture seams; avoid changing every `appURL()` caller implicitly, including PR #9's test and the Electron dispatcher seams. The product runtime also needs the built `views/url-unsupported` assets for the existing external-navigation walk, which currently obtains that view from Vite. Add the missing test-runtime asset copy, not a production routing change. This is a prerequisite for preserving that assertion honestly. -Other suites that construct an in-process `Server`, component fixtures, and Electron-specific module/dispatcher seams are outside this initial conversion set. They are not asserted to require Vite forever. Converting this concrete set addresses the identified product walks without refactoring every browser harness. Existing built-app acceptances remain built-app acceptances. +For new or substantially changed **browser acceptances using `ProductServer`**, prefer its built application unless the assertion needs development modules. This testing directive does not require retroactive conversion of other files or apply to in-process `Server` walks using main's `appURLForServer()`, component fixtures, or Electron module/dispatcher seams. Those boundaries remain as authored. Existing built-app acceptances remain built-app acceptances. ### Remaining development-page exposure -At `b5477211`, a read-only inventory finds 38 e2e test files with a literal `/packages/web/src/index.html` reference and two more using `guerilla-artifact-sharing.helpers.ts`: 40 test files, or 41 source-loading files when that helper is counted. Repository tests that inspect HTML as text are excluded. The proposed 17-file conversion set plus PR #9 leaves 23 of those test files, including the three desktop `appURL()` users. This count is a source inventory, not a measurement of concurrent pages or memory. Component fixtures importing individual modules are a separate exposure and are not counted as full application loads. +At `b5477211`, a read-only inventory found 38 e2e test files with a literal `/packages/web/src/index.html` reference and two more using `guerilla-artifact-sharing.helpers.ts`: 40 test files, or 41 source-loading files when that helper is counted. That historical count is not today's loader count or a measurement of concurrent pages or memory. Repository tests that inspect HTML as text and component fixtures importing individual modules are excluded. The narrowed scope deliberately leaves the following exposures: | Remaining tests | Observed reload/page behavior and why deferred | | --- | --- | +| `channel-product.test.ts`, `channel-crud.test.ts`, `channel-sidebar-remote.test.ts`, `channel-switcher-context-menu.test.ts`, `tab-pages.test.ts`, `keyboard-navigation-product.test.ts`, `artifact-view-wheel.test.ts`, product walks in `top-bar.test.ts` and `auth.01.test.ts` | Deferred from the earlier conversion set. Channel, tab, and keyboard walks include observer pages; the tab walk also reloads. They remain legitimate conversion candidates, but their present product assertions do not require this contribution to change their delivery path. The channel drag flake has no established resource cause. | +| `markdown-content-sync.spec.ts`, `path-artifact-saga.test.ts`, `path-artifact-real-stack-coverage.test.ts`, `navigable-history-real-stack.01/.02.test.ts`, `onboarding-browser.test.ts` | Deferred from the earlier set. These cover real editing, persistence, navigation, or boot/restart; onboarding in particular requires stable-origin continuity. After #32 and recurring cleanup, a larger migration is not justified by the resource evidence. Keep their proof boundaries and assertions intact. | +| Main's web `artifact-isolation.test.ts`, `artifact-move.test.ts`, and desktop `artifact-isolation.test.ts` | Additional `appURL()` users identified by round 2 review on main. They are explicitly outside the two-file conversion. Source inspection finds a reload in web isolation and a second page in web move; no headroom claim is made. | | Web `telemetry-session.test.ts` | One case reloads the application; another opens two tabs in one persistent context, closes it, then opens a new context. This is a concrete remaining multi-page exposure. It uses an in-process server with an injected telemetry sink; converting that boundary needs its own proof review. | | Web `artifact-navigation.01.test.ts`, `artifact-navigation-history.test.ts` | Their `beforeEach` loads the development app, clears storage, then reloads it: two full graph loads per case. They use directly controlled `Store`/`Server` instances. These are sensible next conversion candidates, not proven-safe exceptions. | | Web `auth.02.test.ts` | The silent reconnect case explicitly reloads the app. Other cases inject authentication failures into an in-process server. Reconcile this base-revision inventory against PR #9's merged changes before implementation. | @@ -156,7 +181,7 @@ At `b5477211`, a read-only inventory finds 38 e2e test files with a literal `/pa | Desktop `all-webview-dispatcher.01/.02.test.ts`, `webview-event-parity.test.ts` | The executable-theme parity walk explicitly waits for a top-level document reset and later closes/relaunches Electron. The navigation-history walk quits and relaunches. These remain exposed; application resets, fresh Electron processes, and guest-webview reloads must be distinguished in any follow-up conversion. | | Desktop `artifact-view-webview-wheel.test.ts`, `guerilla-artifact-sharing.test.ts`, `path-webview-content-reload.test.ts`, `artifact-navigation.test.ts`, `native-navigation-key.test.ts`, `artifact-navigation-external.test.ts` | Launch the development application in Electron; several reload or navigate guest documents. Native input, preload, and webview seams are outside this browser-product conversion. No measurement establishes their peak memory. | -Thus this contribution moves suitable product walks onto shipped assets and reduces their module requests; it does not establish a post-#32 memory saving, eliminate unbundled loading, or certify every remaining test's headroom. Preserve the remaining walks in full validation and use the new diagnostics if they fail. The proposed testing directive is a default for new or changed product acceptances, not a false assertion that all existing tests already comply. +This contribution does not establish a post-#32 memory saving, eliminate unbundled loading, or certify every remaining test's headroom. Preserve the remaining walks in full validation and use the new diagnostics if they fail. The testing directive is limited to new or substantially changed `ProductServer` browser acceptances. ### Reconciliation with current main @@ -174,15 +199,15 @@ Read kernel output through a bounded capture and retain at most **64 KiB of rele Periodic sampling, per-process descriptor inventories, cgroup probes, detailed browser/request event records, and browser-revision comparison remain optional investigation tools. They are not new merge obligations or spec requirements in this contribution. Do not add default videos, heap snapshots, or NetLogs. Existing native retry evidence and trace policy remain unchanged. -Write sidecars under the existing checked `.testshards/results` tree, outside disposable scratch, and download them with the other reports. Keep at most one attempt's bounded resource diagnostics per worker. A failed download leaves that attempt locked; it does not start a growing queue of retained runs on the same worker. The next run's results reset is allowed only after retrieval completes and the lock is released. Coordinator-side `.test-runs/` retention is unchanged. This bounds worker storage without expiring undownloaded evidence. No new cross-run archive or retention daemon is needed. +Write sidecars under the existing checked `.testshards/results` tree, outside disposable scratch, and download them with the other reports. Keep at most one attempt's bounded resource diagnostics per worker. A failed download leaves that attempt locked while the coordinator retries retrieval and renews; it does not start a growing queue of retained runs. Normally the next results reset follows download and release. If the coordinator stops renewing past the recovery deadline, automatic recovery may discard those abandoned reports and records that loss. Bound its recovery receipt to 128 KiB, with explicit truncation, within the incoming attempt's 8 MiB diagnostic budget. Coordinator-side `.test-runs/` retention is unchanged. No cross-run archive or retention daemon is needed. ## Authority and derivation After proposal convergence, derive the smallest changes in the existing owners: -- [`testing-policy.md`](../../specs/arch/testing-policy.md): built application as the default for product browser acceptance when development-module behavior is not under test; worker-owned reusable test runtimes and cleanup expectations. -- [`blaxel-testshards.md`](../../specs/arch/test-runner/blaxel-testshards.md): exclusive worker lock through setup, process teardown and result retrieval; no time-based reclamation of uncertain work; recovery and capacity consequences; the actual older-runner behavior; pool-script locking; attempt temporary directories, recurring exact-prefix cleanup and bounded worker retention. Replace its TTL availability escape and shell-exit unlock expectations rather than leaving conflicting requirements. -- [`reporting.md`](../../specs/arch/test-runner/reporting.md): minimum resource observations and artifact references, bounded content, unavailable evidence, and retention alongside failed/recovered native attempts. +- [`testing-policy.md`](../../specs/arch/testing-policy.md): built application as the default for new or substantially changed `ProductServer` browser acceptances unless development-module behavior is under test, excluding in-process `Server`/`appURLForServer()` walks; worker-owned reusable test runtimes and cleanup expectations. +- [`blaxel-testshards.md`](../../specs/arch/test-runner/blaxel-testshards.md): exclusive reservation through setup, process teardown and result retrieval; renewal, automatic recovery after lost renewal, owned-process cleanup before reuse, and the explicit abandoned-report exception; actual older-runner behavior; pool-script locking; attempt temporary directories, recurring exact-prefix cleanup and bounded worker retention. Replace its blind TTL escape and shell-exit unlock expectations rather than leaving conflicting requirements. +- [`reporting.md`](../../specs/arch/test-runner/reporting.md): minimum resource observations and artifact references, bounded content, unavailable evidence, retention alongside failed/recovered native attempts, and recovery receipts that disclose discarded orphan reports. Use existing lifecycle and reporting contracts in `test-runner.md` and `sharded-execution.md`, adding only the cross-references or ownership corrections required by the outer Blaxel lock. Existing isolated-GitHub-only lifecycle fixtures keep their placement. No `preflight.md` delta is planned. Optional investigation instrumentation remains optional. @@ -194,10 +219,10 @@ All execution is on Blaxel at committed, pushed revisions. The shared developmen Derive proofs for the principal paths without a combinatorial fault matrix: -- One run excludes a competing coordinator through preparation, teardown, and a deliberately delayed result download. Passing the former TTL never admits a second run while processes or retrieval remain. Exercise both targeted and planned dispatch through the real provider. -- A lost coordinator, ambiguous dispatch, or failed retrieval keeps the lock held. Recovery after the coordinator has ended stops only attributed processes, retrieves the available evidence, and then permits reuse. Contract tests cover error handling; real seam coverage must include dispatch and retrieval, not just mocked API responses. -- The inspected older runner does not reclaim a new non-expiring held lock. Legacy helper residue is removed on the next updated run. Pool-script mutation respects the same lock. An unexplained test process or ownership record prevents cleanup/dispatch rather than authorizing deletion. -- Exact-prefix and stale-scratch cleanup preserves unrelated files and live consumers. Real runtime builds demonstrate once-per-worker reuse, final disposal, and partial-build cleanup. A Blaxel observation verifies Node temporary paths and Chromium shared-memory descriptors under the attempt's `TMPDIR`. +- One run excludes a competing coordinator through preparation, teardown, and a deliberately delayed result download while renewing. Passing the former TTL never admits a second run beneath that valid reservation. Exercise both targeted and planned dispatch through the real provider. +- Stopping coordinator renewal permits automatic acquisition after the recovery interval, both with a quiet worker and with an owned supervisor/descendant still alive. Reuse follows confirmed termination; the receipt records any discarded reports. Contract coverage uses a controlled clock for renewal/deadline calculation, custom timeout, and error paths. A private-worker seam crosses real dispatch, recovery, and delayed retrieval, including a late dispatch rejected after ownership changes. It must not reduce to mocked API responses or simulate a coordinator crash by killing only the shard. +- Concurrent recovery attempts cannot both acquire; delayed renewal/release cannot alter a successor's reservation. Keep this at the lock helper's real filesystem boundary. The inspected older runner does not reclaim a new held lock with `expiresAt: 0`. Legacy helper residue is removed on the next updated run; ambiguous legacy ownership or failed process cleanup holds and reports. Pool-script mutation respects the same acquisition/recovery path. +- Exact-prefix and stale-scratch cleanup preserves unrelated files, live consumers, and the current attempt directory, while removing earlier scratch even when its metadata is incomplete. Real runtime builds demonstrate once-per-worker reuse, final disposal, and partial-build cleanup. A Blaxel observation verifies Node temporary paths and Chromium shared-memory descriptors under the attempt's `TMPDIR`, recording the actual browser version used. - A failed or recovered acceptance retains bounded start/end/OOM evidence through retrieval, including when its worker exits. Contract coverage handles unavailable probes, unchanged original test outcomes, truncation, and limits. The live recovery checks use small, disposable, owned processes on private workers. No deliberate OOM, process storm, or invocation of the repository's isolated-GitHub-only destructive fixtures is authorized. If a required assertion cannot be proven safely within Blaxel-only execution, report that specific blocker at proof review; do not silently substitute local or GitHub execution. @@ -208,7 +233,9 @@ Re-run the converted acceptances without weakening their user-visible assertions For the final gate, provision a separate sandbox named **`tv-cleanup-coordinator-rose`**, excluded from testshard pool labels. It will fetch the committed, pushed branch and run `npm run verify -- blaxel`, so the invoking machine's lint/type-check/package-manifest phases also execute on Blaxel. The two experimental workers and coordinator are a plan, not currently provisioned capacity. Provision and verify their identities before starting validation; if unavailable, report blocked rather than run anything on this host or silently substitute GitHub execution. The user has already authorized outside-pool Blaxel hosts. -The experiment plan adds at most three sandboxes. The spec's pool-size and tier examples are not a current capacity inventory: inspect actual workspace usage and provider limits before provisioning/dispatch. Do not claim slots are reserved or alter shared workers to obtain them. Delete this contribution's experiment workers through the pool script after retrieving their reports and before final standard 36-shard verify, leaving only its coordinator. That final run uses the normal `poc` path without intentional contamination or interruption, after old in-flight runs have finished and active coordinators carry the lock correction. Archive its reports before disposing the coordinator. Record the exact tested revision, failures, recovered flakes, and any measurement limits. Ready-PR CI remains the repository's merge requirement; the contribution's execution evidence stays on Blaxel as instructed. +Supply the coordinator's credentials through the Blaxel SDK's `process.exec` environment: `BL_WORKSPACE` and an authorized `BL_API_KEY` for its outbound Blaxel calls, plus `BLAXEL_TV_GH_TOKEN` for repository access. The launching process reads the existing authorized secrets into memory and passes them directly in that environment object; do not interpolate their values into shell commands, uploaded scripts, output, or reports. The inspected SDK accepts the Blaxel pair without a CLI login/config file, and `readBlaxelGithubToken()` accepts `BLAXEL_TV_GH_TOKEN` when no `.blaxel-gh-token` file is present. Use that environment path in the fresh coordinator, with `GIT_ASKPASS` reading the environment for its initial fetch; the existing runner passes only the GitHub token onward to workers as `GH_TOKEN`. Do not copy local credential files or persist tokens in the coordinator's filesystem, sandbox configuration, or Git configuration. Confirm that the required credential is available and valid for the run before dispatch; its availability is not established by this proposal. End the credential-bearing process after downloading its reports and dispose the dedicated coordinator. This describes ephemeral process injection, not a claim about provider-internal retention. + +The experiment plan adds at most three sandboxes. The spec's pool-size and tier examples are not a current capacity inventory: inspect actual workspace usage and provider limits before provisioning/dispatch. Do not claim slots are reserved or alter shared workers to obtain them. Delete this contribution's experiment workers through the pool script after retrieving their reports and before final standard 36-shard verify, leaving only its coordinator. That final run uses the normal `poc` acquisition path without intentional contamination or interruption; it does not require a pool-wide wait for other runs or a coordinated update of other branches. Archive its reports before disposing the coordinator. Record the exact tested revision, failures, recovered flakes, and any measurement limits. Ready-PR CI remains the repository's merge requirement; the contribution's execution evidence stays on Blaxel as instructed. Keep browser-version alignment, shard count, browser request limits, retries, and production memory behavior outside this contribution. Forced GC is an investigation tool, not a proposed mitigation. The demo-mode and drag flakes have not been proven to share the resource-exhaustion cause. @@ -216,4 +243,6 @@ Keep browser-version alignment, shard count, browser request limits, retries, an Proposal round 1 at `1652787e`: FAIL on lease scope and mixed-version consequences. The response at `99b7ed93` narrowed the proposal while those human scope decisions were pending. Josh's 2026-10-27 instruction brings lock correction and recurring legacy cleanup into this PR and removes the pool-recreation choice. The [review response](test-harness-resource-cleanup-review-response.md) records the current dispositions without changing the independent review's historical verdict. -Revised proposal: awaiting independent re-review. No spec, proof, test, or implementation edits yet. No tests, builds, remote experiments, or worker operations were performed for this revision. Unknowns are live provider behavior during failed dispatch/recovery, actual `TMPDIR` placement and post-mitigation memory savings, current capacity, and the availability impact of locks retained after failures. Josh's calibration limits lock work to our runner and pool scripts. His spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. +Proposal round 2 at `d3570bf4`: FAIL because dead-coordinator lock recovery is undefined. The [round 2 review](test-harness-resource-cleanup-review-r2.md) is preserved unchanged. Josh selected automatic recovery, required that an orphaned active run cannot hold a worker forever, and delegated the recovery mechanism and built-app conversion scope. His supervisor clarified that the suggested predicate and grace-period wording were illustrations, not requirements. This revision proposes one renewable recovery deadline, retains the ownership checks needed for safe reuse, narrows conversion to two files, and addresses refinements 2–4. It is ready for independent re-review, not converged. + +No spec, proof, test, or implementation edits yet. No tests, builds, remote experiments, or worker operations were performed for this revision. Unknowns are live provider behavior during failed dispatch/recovery, today's worker browser versions, actual `TMPDIR` placement and post-mitigation memory savings, current capacity/credential availability, and observed recovery timing. The known cost is up to the configured worker timeout plus 15 minutes after the last renewal, then acquisition and cleanup; genuinely failed cleanup or unidentifiable legacy work still needs repair. Josh's calibration limits lock work to our runner and pool scripts. His spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. diff --git a/docs/working/test-harness-resource-cleanup-review-r2.md b/docs/working/test-harness-resource-cleanup-review-r2.md new file mode 100644 index 00000000..a5550396 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-review-r2.md @@ -0,0 +1,72 @@ +# Proposal review, round 2: test-harness resource cleanup (draft PR #12, d3570bf4) + +**Verdict: FAIL, on one blocker.** The proposal now follows Josh's decisions, its evidence still holds, and its scope is much closer to its benefit. What remains is a hole in the lock design it was asked to make correct: it says how a worker becomes locked but not how a worker locked by a dead coordinator ever becomes usable again. Four refinements follow, one of them a decision for Josh. + +- Reviewed: `docs/working/test-harness-resource-cleanup-proposal.md`, the review response and the unchanged investigation report at `d3570bf4`, against the reviewer checklist, Josh's decisions as the response records them, the code at the branch base `b5477211`, and current `origin/main` (`a8e73fbf`), including PR #32. +- I ran nothing. Every check is a read of files in the checkout or of main's history. + +## What the revision gets right + +- **Round 1's blockers are resolved as Josh decided.** The lock work is now Josh's requested correction, not a redesign presented as a prerequisite. Old branches are handled by removing the two known prefixes on every run under the lock, which I checked against the old runner: it reclaims a lock only when `expiresAt` is positive and past (`scripts/run-blaxel-testshards.mjs:406`), so a new lock written with `expiresAt: 0` is left alone, as the proposal says. +- **The relative benefit is stated** (round 1 refinement 1): the historical 3.2 GB test cost against 1.16 GB and 0.079 GB of leaked prefixes on the two inspected workers, with the other 2 to 3 GB of baseline named as out of scope. +- **Diagnostics are reduced to the minimum** that would have identified the incident, with the sampler and descriptor inventories made optional (refinement 3). +- **The infrastructure is named**: a private two-worker pool for experiments and a coordinator sandbox for the final gate, never the shared `poc` pool for experiments (refinement 4). +- **Remaining exposure is listed** with per-file behaviour (refinement 5). I spot-checked four of those claims at `b5477211` and all hold: the telemetry walk's two tabs and reload, the artifact-navigation `beforeEach` reload, the silent-reconnect reload in `auth.02`, and the reconnect test's count of top-level navigations. +- **The PR #32 assessment is honest.** It stops claiming the conversions repair the measured leak, and it requires any resource comparison to use a baseline containing #32. +- **Factual claims check out**: PR #9 is `f81a2465` on main; the pool script deletes through the provider API (`scripts/manage-blaxel-testshards.mjs:76`) and inspects through `process.exec` (lines 94, 115); the download race is unchanged on main, whose coordinator diff since the base only moves the checkout and dependency scripts into a module. + +## Blocker: recovery of a locked worker is not defined + +Josh asked for a lock that is correct for our own runner and pool scripts. The proposal removes time-based reclamation and holds the lock through teardown and download. That is right as far as it goes. It then says (lines 70 and 72): + +- "A coordinator crash can leave a lock indefinitely." +- "Once the original coordinator is confirmed ended, recovery reuses the same stop/audit, download, and matching-lock release steps." +- "If execution status, process teardown, or retrieval cannot be confirmed, leave the lock held and report the specific reason. No timer or blind `rm -rf` is a recovery procedure." +- An expired legacy lock "is not reclaimed merely because of its age; uncertain ownership needs the same recovery checks." + +What is missing: + +1. **No entry point.** Nothing says what runs recovery: a runner command, a pool-script command, or a person following a procedure. Spec derivation would have to invent it. +2. **"Confirmed ended" has no method.** The lock records the coordinator's host and PID (`lockCommand`, line 396). A recovery run on the same host can check that PID; most coordinators do run on this host, so that covers most crashes. From any other host there is no way to confirm it. The proposal should say which evidence suffices: a dead coordinator PID on the same host, or an operator's assertion otherwise. +3. **Undownloaded results of a dead coordinator are never released.** The results exist to be downloaded by the coordinator that asked for them. If it is gone, recovery can retrieve them to a run directory nobody will read, or discard them. Either is fine; the proposal has to choose, because "until results have been downloaded" otherwise means forever. +4. **The cost is stated but not sized.** Every coordinator killed without its signal handler running (`kill -9`, a host reboot, a lost session) removes a worker from the shared pool until someone runs recovery. With many agents sharing one pool, that is a steady drain. The old design healed this in twenty minutes; the new one heals it never, by itself. + +A correct and simple design within Josh's calibration is available and worth putting to him as the decision it is: reclaim a held lock automatically when worker-local evidence shows the run is over (the recorded shard supervisor is dead by PID and start time, the owner audit finds no live or stale owners, and results are either marked downloaded or older than a generous grace period), and otherwise hold and report. That uses only our own records, keeps the pool self-healing, and loses nothing but the reports of a coordinator that died before reading them. If Josh prefers manual recovery, the proposal can say so, with the entry point and evidence above. + +## Refinements (none blocking) + +1. **The built-application conversion needs a decision from Josh, with the post-#32 picture in front of him.** + - PR #32 removes the mechanism the investigation measured: Vite services no longer answer `304`, so a reload no longer holds about 465 MB. The proposal says so and keeps the 17-file conversion on two other grounds, product fidelity and fewer requests, while admitting their benefit is unmeasured. + - Main has moved the other way for in-process servers. Twenty-one test files and a shared helper now reach the application through the new `appURLForServer()` function, which puts a test's own server behind the Vite proxy on purpose (`test/helpers/product-server.ts`, main). The proposed testing-policy directive, built application by default for product browser acceptance, would read as contradicting that unless it is scoped to `ProductServer` walks. + - Josh said to keep "suitable" built-app acceptances. The proposal should offer him the choice plainly: the full set on the fidelity argument, or the walks whose proofs name the built application or that reload several pages, with the rest left to a later contribution. Either way, scope the directive. + - The conversion table is a base-revision inventory. On main, `artifact-isolation.test.ts` and `artifact-move.test.ts` in the web suite and `artifact-isolation.test.ts` in the desktop suite also use `appURL()`. The proposal defers reconciliation to implementation, which is acceptable, but it inspected main at `a8e73fbf` and could name them now. + +2. **The runner's own scratch is treated more cautiously than the legacy prefixes.** Legacy `television-*` directories are removed on the strength of the lock and the idle check. The runner's own `/tmp/tv-testshards/` directories, which are more certainly ours, additionally need intact metadata and "confirmed completed/recovered ownership", and "missing or inconsistent metadata needs recovery". A crashed run with half-written metadata therefore leaves its scratch until the manual recovery above. Josh's decision covers "stale per-run scratch". Under the lock, after the owner audit confirms no live or stale owners, remove every attempt directory except the current one. That is what "no adversarial design" allows. + +3. **Two statements about old coordinators ask for more than is needed.** + - "Carry the small lock lifecycle correction into branches still dispatching to the shared pool" and the final verify's precondition that "active coordinators carry the lock correction". Branches take the correction when they take main; nothing can make that happen sooner, and nothing requires it. A new run's non-expiring lock already keeps old coordinators off its worker. The only effect of an old coordinator on a new run is that its workers may show expired legacy locks the new runner will not reclaim, which costs capacity, not correctness, and is another reason to settle the blocker. + - "After old in-flight runs have finished" as a precondition for the final run cannot be known on a shared pool. Drop it. + +4. **Two smaller points.** + - The investigation says the workers run Chromium 148 through the coordinator's build alias; PR #32's spec attributes the `304` behaviour to Chromium 145 and says other versions may differ. Both documents are in the tree. Say which version the workers actually run, or that it is unknown, so the proofs measure the right thing. Browser-version alignment itself can stay out of scope. + - The final gate runs `npm run verify -- blaxel` from a coordinator sandbox inside Blaxel. That sandbox needs the Blaxel credentials and the GitHub token the runner reads; the proposal should say how they get there without being persisted. + +## Reviewer checklist + +The proposal changes no spec yet. On the items that apply now: + +- **Authority.** The derivation section names the right owners, and now says the Blaxel spec's TTL escape and shell-exit unlock are replaced rather than left in conflict. Good. +- **Testing directives.** The one proposed directive is the place where refinement 1's scoping belongs. +- **Writing.** The document is long for a proposal, but the length is in the lock and cleanup sections that Josh asked to be decided here. The remaining-exposure table is the right shape: per file, what was seen and why it is deferred. +- **Status.** The review-status section is accurate and does not claim convergence. + +## Evidence and limits + +- Read the proposal, the response and the investigation in full; diffed the copied round 1 review against mine (identical) and the investigation against its round 1 version (unchanged). +- Read the lock, trap and timeout code at the base and main's coordinator diff; the pool script; main's `product-server.ts` and `playwright.ts` changes; PR #32's commit, spec and proof text; and the four spot-checked tests at `b5477211`. +- Not established: whether Blaxel kills a shard process at its timeout; which Chromium build the workers run today; main's current count of development-page loaders beyond the `appURL()` and `appURLForServer()` lists. + +## Housekeeping + +- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel. `git fetch origin` updated remote-tracking refs. +- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 277a25fb..4bf5f389 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -6,28 +6,30 @@ Independent round 1 reviewed `1652787e`. Reviewer: Claude Fable (Nina), arranged All of the following belong in PR #12: -- Keep a worker exclusively reserved while the previous run's processes may be alive and until results have been downloaded. Account for direct Blaxel API callers as well as shard coordinators. +- Keep a worker exclusively reserved through process cleanup and result download. Account for our pool script's direct Blaxel API calls as well as shard coordinators. Automatic recovery of an abandoned run is the explicit exception to download-before-release described below; recovery must stop its owned processes before reuse. - After taking that lock, every run removes the known `television-source-cli-e2e-*` and `television-web-bundle-*` directories directly under `/tmp`, plus stale per-run scratch. Preserve everything else. This handles residue from test revisions without the helper fixes. - Keep worker-lifetime helper cleanup, per-run scratch, suitable built-app acceptances, and bounded failure diagnostics. Drop pool drain/recreation and the pending rollout choice. Josh then calibrated the lock scope: only our runner and pool scripts touch these workers. Keep their locking correct and simple; do not defend against hypothetical callers or add elaborate mixed-version machinery. The revised proposal uses the existing lock directory and ownership identity, holds it through teardown/download, and makes pool operations respect it. It does not introduce a persistent compatibility fence, separate worker controller, or general API gateway. +Following round 2, Josh selected automatic recovery and required that a crashed coordinator's still-active run cannot hold a worker forever. The supervisor clarified that his suggested predicate and grace-period mechanism were illustrations: keep a special case if correctness needs it and explain why. Josh also delegated the built-app scope judgment, noting that recurring cleanup reduces the case for a broad conversion. The resulting design and scope below are this contribution's proposal, not a claim that Josh prescribed their mechanics. + The [proposal](test-harness-resource-cleanup-proposal.md) incorporates the result-download-race note. Its separate file is removed because the correction is now in scope. No spec, proof, or implementation has been changed. ## Blockers | Finding | Current assessment and disposition | | --- | --- | -| B1: lease redesign is unnecessary for scratch cleanup alone, with unresolved availability and mixed-version consequences | The original scope objection was warranted. Josh now explicitly requires a simple correction to our lock lifecycle. Keep atomic directory acquisition and `lockId`; remove age-only reclamation, audit descendants, and release from the coordinator after downloads. Cover dispatch uncertainty and signal/exception paths using the existing cleanup machinery. State the availability cost: uncertainty can hold a worker indefinitely. Keep the fixed results tree and protect it through download instead of adding a cross-run artifact archive. | -| B2: old branches keep leaking outside the new directory | Address this with Josh's selected recurring pre-run deletion. New locks use `expiresAt: 0`, which the inspected old acquisition code does not reclaim. Older coordinators still release their own locks before downloading, so active dispatching branches need the small lifecycle correction and old in-flight downloads must finish. No permanent old-client exclusion, separate controller architecture, or pool recreation is proposed. Helpers on older branches can still leak; subsequent updated runs reclaim their known residue. | +| B1: lease redesign is unnecessary for scratch cleanup alone, with unresolved availability and mixed-version consequences | The original scope objection was warranted. Josh now explicitly requires the lock correction. Keep atomic directory acquisition and `lockId`, replace blind reclamation with renewal and cleanup before reuse, audit descendants, and normally release after download. Round 2 below supplies automatic recovery and sizes the availability cost. Keep the fixed results tree rather than a cross-run artifact archive. | +| B2: old branches keep leaking outside the new directory | Address this with Josh's selected recurring pre-run deletion. New locks use `expiresAt: 0`, which the inspected old acquisition code does not reclaim. Older coordinators still release their own locks before downloading; branches take that correction when they integrate main. Round 2 refinement 3 removes the requirement to update all active branches or wait for all old in-flight runs before this contribution's final verification. No permanent old-client exclusion, separate controller architecture, or pool recreation is proposed. Helpers on older branches can still leak; subsequent updated runs reclaim their known residue. | ## Refinements | Finding | Current disposition | | --- | --- | -| R1: quantify relative benefit | Retain the historical roughly 3.2 GB shared-memory increase and 1.16 GB / 0.079 GB leaked-prefix measurements. Explicitly assess PR #32: it directly addresses 304-buffer accumulation for registry Vite services. Keep suitable built-app acceptances for shipped-asset coverage and fewer module requests, with no claim that they are necessary to fix that leak or will save the historical amount of memory. Any incremental resource measurement uses a baseline containing #32. | +| R1: quantify relative benefit | Retain the historical roughly 3.2 GB shared-memory increase and 1.16 GB / 0.079 GB leaked-prefix measurements. PR #32 directly addresses 304-buffer accumulation for registry Vite services. Narrow conversion to two files for their proof boundary and repeated loads; do not claim they are necessary to fix that leak or will save the historical amount of memory. Any incremental resource measurement uses a baseline containing #32. | | R2: compare pool recreation with legacy deletion | Josh chose recurring exact-prefix cleanup. Remove the alternatives table, recreation recommendation, and unresolved rollout decision. Require whole-worker exclusion and idle confirmation before deletion; the prefix alone is insufficient evidence of disuse. Preserve unrelated `/tmp` contents and persistent caches. | -| R3: commit only to minimum diagnostics | Keep parent start/end RAM, shared memory, temporary free space and OOM-counter observations, plus bounded kernel OOM lines on failure. Retain evidence for recovered retries. Keep the 128 KiB/surface and 8 MiB/attempt limits. Hold one attempt's evidence on a worker until retrieval; a blocked worker cannot accumulate new attempts. Detailed probes remain optional. | +| R3: commit only to minimum diagnostics | Keep parent start/end RAM, shared memory, temporary free space and OOM-counter observations, plus bounded kernel OOM lines on failure. Retain evidence for recovered retries. Keep the 128 KiB/surface and 8 MiB/attempt limits. Hold one attempt's evidence until retrieval or abandoned-run recovery; disclose discarded orphan reports in a bounded receipt. Detailed probes remain optional. | | R4: name infrastructure and account for capacity | Retain two private `rose-cleanup` workers and the separate Blaxel coordinator as planned infrastructure, not reserved capacity. Recheck actual pool inventory and provider limits rather than assuming the old 72+3 arithmetic describes today's workspace. All test/build execution remains on Blaxel; private experiments use bounded owned processes and do not invoke isolated-GitHub-only destructive fixtures. | | R5: disclose remaining unbundled exposure | Retain the named inventory, labeled with its original base revision, and require reconciliation against current main before implementation. Existing deferred cases remain visible; no claim that all of them require Vite or have sufficient headroom. | | R6: justify helper fixes in addition to scratch | Keep helper cleanup for within-shard worker disposal and callers without a Blaxel wrapper. Preserve once-per-worker/per-version build reuse and clean failed builds. This contribution does not authorize cleanup on the shared development host. | @@ -38,3 +40,29 @@ The [proposal](test-harness-resource-cleanup-proposal.md) incorporates the resul Independent re-review should assess the revised scope against Josh's decisions and simplicity calibration: our acquire/release/error paths, result retrieval before release, recovery of uncertain work, and the pool-management script. There is no adversarial API-caller requirement. Live provider behavior and `TMPDIR` placement still need proof-stage experiments. No observed Chromium failure has been attributed to the lock or download races, and no post-#32 experiment establishes the remaining memory benefit of built loading. Validation of this revision consists of document, code, policy, and PR inspection plus Git whitespace/diff checks. No tests, builds, browsers, servers, remote experiments, or worker cleanup were run for this revision. The pushed proposal is the next independent-review checkpoint; spec derivation follows convergence. + +## Round 2: automatic recovery and narrower conversion + +Independent round 2 reviewed `d3570bf4`. The [full review](test-harness-resource-cleanup-review-r2.md) is preserved verbatim: **FAIL on undefined recovery of a dead coordinator's lock**. The blocker was warranted. A lock that survives coordinator death without an automatic recovery entry point would steadily remove pool capacity. The proposal now supplies all four missing decisions: + +| Blocker detail | Proposed resolution | +| --- | --- | +| Entry point | The next canonical `acquireAny()` call invokes the common worker lock helper. Pool mutations reuse it. No new recovery service or manual routine for ordinary coordinator crashes. | +| Evidence of coordinator loss | The coordinator renews once a minute through setup, execution, cleanup, and download. A worker-local deadline is the last renewal plus the configured worker timeout plus 15 minutes. Missing renewals beyond that interval mean ownership has been abandoned, whether caused by a crash, suspension, or network partition; no claim of cross-host PID certainty. Expiry permits cleanup, not reuse beneath survivors. | +| Undownloaded reports | Normal runs hold through verified download. After lost renewal reaches the recovery deadline, discard orphan reports during reuse and record their loss with the old identity, process cleanup, and deadline in a bounded recovery receipt. No orphan archive or additional blocking download. A returning coordinator that lost ownership reports infrastructure failure. | +| Capacity cost | At the 480-second cold-worker default, an orphan becomes recoverable 23 minutes after its last renewal; the 240-second default gives 19 minutes. Actual reuse adds the next acquisition attempt and bounded cleanup. A still-running owned supervisor or descendant is stopped by token and audited, so it cannot renew the lock or prevent ordinary recovery forever. | + +One recovery sequence handles quiet and still-active abandoned attempts. The proposal deliberately forgoes early quiet-worker recovery to avoid a second liveness test or report-age clock. It keeps a short OS-released filesystem critical section for competing acquisition/renewal/release, and checks ownership at remote dispatch registration so a delayed API call cannot start obsolete work after recovery. These are races in our own code paths, not defenses against hypothetical callers. + +Two limits remain for correctness: a process whose identity or termination cannot be confirmed cannot be ignored, and a legacy lock without outer-run attribution cannot justify killing by name. Hold and report those repair cases. Updated, attributable runs recover automatically even when their supervisor remains alive. Empty acquisition records with no dispatched work can be reclaimed after a clean worker audit; missing scratch metadata never adds a gate. This distinguishes a real cleanup failure from an ordinary crashed coordinator without multiplying recovery procedures. + +Refinement 1 is resolved by contributor judgment: **five cases in two files**, the settings-persistence and executable-lifecycle cases in `theme-product-acceptance.test.ts` and all three cases sharing `browser-demo-mode.test.ts`'s page helper. The theme product and delivery proofs already describe built application behavior; both files include reload/restart walks and reuse existing helpers. PR #9 owns reset. After PR #32 and routine cleanup, the remaining 15-file migration has diminishing benefit relative to its proof and regression work. Name main's additional isolation/move callers as deferred too. Limit the proposed default to new or substantially changed `ProductServer` browser acceptances, leaving in-process `Server` / `appURLForServer()` paths and Electron seams alone. + +| Refinement | Assessment and local change | +| --- | --- | +| 2: runner scratch needs stronger metadata than legacy prefixes | Accepted. After taking the lock and obtaining a clean worker owner audit, remove every earlier attempt directory in `/tmp/tv-testshards/` except the current one. Missing or partial directory metadata alone cannot block cleanup. Keep the existing path boundary and live-process protections. This does not decide how another run's held lock is recovered. | +| 3: require other branches to update and old runs to finish | Accept removing both pool-wide prerequisites. `expiresAt: 0` protects updated runs from the inspected old acquisition code; only the new helper understands the renewable deadline. Branches integrate the fix normally and final verify uses ordinary acquisition. An old coordinator still releases before downloading, which new code cannot retroactively protect. Respect unexpired legacy TTLs; after expiry a clean worker audit permits reclamation, while unattributable live work holds and reports. | +| 4a: distinguish Chromium 148 investigation from PR #32's 145 evidence | Accepted. State the observed trace version `148.0.7778.96`, the Playwright manifest's expected `145.0.7632.6`, and main's conditional 1223-to-1208 cache aliases. Today's actual worker browser version is unknown. Record the launched browser version and resolved executable/cache path during future Blaxel validation before interpreting memory or temporary-file evidence. No browser alignment change or new experiment is proposed here. | +| 4b: credentials for the Blaxel coordinator sandbox | Accepted. Specify process-environment injection of `BL_WORKSPACE`, an authorized `BL_API_KEY`, and `BLAXEL_TV_GH_TOKEN`, loaded in memory on the launching host. The installed SDK's authentication source and repository token reader support these paths. Do not copy credential files, embed values in commands/scripts/logs, or persist them in sandbox/Git configuration; verify availability before dispatch and end the credential-bearing process after report retrieval. No secret was read, copied, or transmitted while preparing this revision. | + +The revision is ready for independent re-review; the historical FAIL is unchanged until that review converges. No spec, proof, or implementation was edited. No test, build, browser/server launch, remote experiment, or worker operation ran. Live renewal/recovery behavior, browser versions, `TMPDIR` placement, and incremental resource benefit remain unmeasured; the proposal names the private Blaxel validation needed after spec/proof convergence. From cd8aa453ad0b8ad696b9d872ce14e57956754e1d Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:45:42 +0000 Subject: [PATCH 05/64] Specify worker resource cleanup and abandoned-run recovery Derive the spec checkpoint from the converged cleanup proposal. Reserve Blaxel workers through owned-process teardown and report retrieval, recover abandoned runs after lost renewal, and restrict temporary-file cleanup to known test leftovers and per-attempt scratch. Define bounded resource evidence, reusable worker build lifetimes, and the scoped built-app default. Apply the independent proposal review's bounded refinements: one 20-minute recovery interval using the existing option, an explicit deletion path for abandoned workers that fail cleanup, legacy-work skipping, and provisioning exclusion. Preserve the review and record the derivation and its limits. No proofs or implementation changes. Source/policy review and Git diff checks only; no tests or builds ran. Independent spec review is next, with Josh's spec acceptance at PR time and execution evidence later on Blaxel. --- .../test-harness-resource-cleanup-proposal.md | 38 +++++----- ...test-harness-resource-cleanup-review-r3.md | 70 +++++++++++++++++++ ...arness-resource-cleanup-review-response.md | 33 ++++++++- specs/arch/test-runner/blaxel-testshards.md | 65 ++++++++++++++--- specs/arch/test-runner/reporting.md | 19 ++++- specs/arch/test-runner/sharded-execution.md | 4 +- specs/arch/test-runner/test-runner.md | 3 +- specs/arch/testing-policy.md | 4 ++ specs/index.md | 10 +-- 9 files changed, 208 insertions(+), 38 deletions(-) create mode 100644 docs/working/test-harness-resource-cleanup-review-r3.md diff --git a/docs/working/test-harness-resource-cleanup-proposal.md b/docs/working/test-harness-resource-cleanup-proposal.md index abf3839b..621ef5c9 100644 --- a/docs/working/test-harness-resource-cleanup-proposal.md +++ b/docs/working/test-harness-resource-cleanup-proposal.md @@ -1,6 +1,6 @@ # Test-harness resource cleanup -Proposal for independent re-review, revised 2026-10-27 following Josh's scope decisions. Owner: rose. Branch: `thopter/test-harness-resource-cleanup`, draft PR #12, originally based on `b5477211`. Current `origin/main` was inspected at `a8e73fbf`; it has not yet been integrated into this branch. The supervisor arranges independent review of the proposal, specs, proofs, and implementation; Josh reviews every spec delta at PR time. No production behavior changes are proposed. +Converged proposal, revised 2026-10-27 with round 3's bounded refinements. Owner: rose. Branch: `thopter/test-harness-resource-cleanup`, draft PR #12, originally based on `b5477211`. Main at `a8e73fbf` was integrated in `74d22941` before spec derivation. The supervisor arranges independent review of the proposal, specs, proofs, and implementation; Josh reviews every spec delta at PR time. No production behavior changes are proposed. ## Problem and intended result @@ -10,7 +10,7 @@ The fixes have different effects. The historical theme-reset walk added roughly This contribution fixes the cached runtime lifetimes, moves five acceptance cases in two files onto the built application, and retains compact resource evidence when an acceptance fails. It also makes the worker reservation cover preparation, execution, process teardown, and result download, with automatic recovery when a coordinator stops renewing its reservation. After taking that lock and confirming the worker is idle, each run removes the two known leaked-runtime prefixes and stale run scratch. This repeatedly reclaims residue from older test revisions as well as newer ones. Cleanup preserves provider services, deliberate caches, and the current run; abandoned runs and reports have the bounded recovery rule below. -PR #9 merged as `f81a2465` on 2026-10-01 and owns the built-application conversion of **“executable theme reset reloads connected Chromium documents under confirmed state.”** Reuse that change when integrating main; do not duplicate it. Main also includes `982e564d` (PR #32), making registry Vite services answer conditional requests in full. Its runner spec records Chromium 145 retaining about 2 MB per 304 response, roughly 465 MB per application reload. That newer mitigation changes the baseline for future measurements. The investigation numbers above describe the historical runs, not current main's residual memory cost; neither the helper leak nor the lock/download gaps is repaired by the Vite response change. +PR #9 merged as `f81a2465` on 2026-10-01 and owns the built-application conversion of **“executable theme reset reloads connected Chromium documents under confirmed state.”** Preserve that integrated change; do not duplicate it. Main also includes `982e564d` (PR #32), making registry Vite services answer conditional requests in full. Its runner spec records Chromium 145 retaining about 2 MB per 304 response, roughly 465 MB per application reload. That newer mitigation changes the baseline for future measurements. The investigation numbers above describe the historical runs, not current main's residual memory cost; neither the helper leak nor the lock/download gaps is repaired by the Vite response change. ## Evidence available before implementation @@ -49,7 +49,7 @@ Josh has selected routine cleanup under an exclusive worker lock. There is no po The canonical coordinator intends one shard job per sandbox. [`acquireAny()` and `lockCommand()`](../../scripts/run-blaxel-testshards.mjs) acquire `/tmp/tv-testshard.lock`; another coordinator skips an unexpired lock. Within a shard, surface tasks execute sequentially, but a surface can have several test workers and many child processes. This does not prove that a worker never has overlapping jobs: -1. Lock expiry allows reclamation based on time alone. The record identifies the coordinator host/PID, not the remote shard supervisor. Setup is not fully covered by surface owner tokens. The configured TTL is 900/1200 seconds against a 240/480-second worker timeout, but provider timeout is not evidence that every descendant has stopped. The current spec explicitly uses TTL as the final availability bound for uncertain cleanup. +1. Lock expiry allows reclamation based on time alone. The record identifies the coordinator host/PID, not the remote shard supervisor. Setup is not fully covered by surface owner tokens. The configured TTL is 900/1200 seconds against a 240/480-second worker timeout, but provider timeout is not evidence that every descendant has stopped. Main's spec at `a8e73fbf` explicitly uses TTL as the final availability bound for uncertain cleanup; this contribution replaces that requirement. 2. The shard shell's `EXIT` trap releases the lock before `runShard()` downloads results from `/workspace/television/.testshards/results`. A following checkout clears that fixed tree. This leaves an unprotected interval in which the previous job's evidence can disappear or be replaced. Report identity checks can detect some mismatches but cannot restore deleted files. This source-level race is present at both the branch base and inspected main; no collision was observed during the Chromium investigation. 3. Direct Blaxel process API calls bypass this repository lock. The inventory used that API. `manage-blaxel-testshards.mjs` also provisions and checks workers through `process.exec`, and deletes them through the provider API. The idle snapshots establish neither provider serialization nor a history-wide absence of overlap. @@ -75,7 +75,7 @@ All runner exit paths must follow that order, including acquisition rollback, ar **A coordinator keeps its reservation by renewing it; after renewal stops for the recovery interval, the next acquirer stops and audits that run's owned processes, clears its abandoned state, and takes the lock.** The same procedure handles an already quiet worker and a surviving shard supervisor or test descendant. An expired deadline alone never makes the worker available. -Use the coordinator to renew once a minute, including during result download. Each successful renewal records worker-local time and sets `recoverAfter` to that time plus **the attempt's configured worker timeout plus 15 minutes**. The cold-worker default of 480 seconds therefore permits recovery 23 minutes after the last renewal; 240 seconds gives 19 minutes. A longer configured timeout gets the same margin. Record that timeout when acquiring the lock, so custom runs and time spent before dispatch are covered. Renewal requires the same owner and a still-valid deadline; it cannot revive an expired reservation. Only the coordinator renews: a surviving supervisor cannot preserve a crashed coordinator's reservation forever. No daemon, cross-host PID probe, or separate heartbeat service is needed. +Use the coordinator to renew throughout ownership, including result download. Each successful renewal records worker-local time and sets `recoverAfter` to that time plus **20 minutes**, independent of the worker execution timeout. Reuse `--lock-ttl-seconds` to override this recovery interval; do not add another option or carry the worker timeout in the lock record. Renew comfortably within the configured interval, with cadence derived from it so short private-worker checks work too. Renewal requires the same owner and a still-valid deadline; it cannot revive an expired reservation. Only the coordinator renews: a surviving supervisor cannot preserve a crashed coordinator's reservation forever. No daemon, cross-host PID probe, or separate heartbeat service is needed. The automatic entry point is the canonical runner's `acquireAny()` / lock helper. On finding an expired `recoverAfter`, it performs this bounded worker-local sequence before publishing a new owner: @@ -84,9 +84,9 @@ The automatic entry point is the canonical runner's `acquireAny()` / lock helper 3. Remove the abandoned attempt's scratch and the allowed legacy leftovers, then reset its results as part of the next run's normal preparation. **Discard undownloaded results of this abandoned attempt after the recovery interval.** Do not create a second archive or require an orphan-report download that can itself block reuse. Record the prior worker/run/lock, last renewal and deadline, stopped process identities, cleanup outcome, and whether reports were discarded in a bounded recovery receipt with the incoming attempt's setup sidecars, outside the old results tree being reset. Loss of reports never becomes a passing result for the abandoned run. 4. Publish the incoming reservation only after cleanup succeeds. Recovery is repeatable: if its process dies midway, the old expired identity remains until another acquirer finishes the same work. Normal release and renewal check `lockId`; a returning coordinator that has lost it must stop work and report infrastructure failure, never release or mutate the new owner's attempt. -This defines coordinator loss operationally: it has stopped renewing for longer than a complete worker timeout plus a generous margin. It does not prove that a remote OS process died. A network partition or suspended coordinator beyond that interval has the same consequence as a crash. A healthy coordinator keeps renewing through a long download, so elapsed shard time alone does not end retrieval protection. Once renewal is lost, orphaned reports are the explicit exception to normal download-before-release. Their file timestamps do not extend the deadline; a hung child must not defeat recovery by continuing to write. +This defines coordinator loss operationally: it has stopped renewing for the recovery interval. It does not prove that a remote OS process died. A network partition or suspended coordinator beyond that interval has the same consequence as a crash. A healthy coordinator keeps renewing through a long download, so elapsed shard time alone does not end retrieval protection. Once renewal is lost, orphaned reports are the explicit exception to normal download-before-release. Their file timestamps do not extend the deadline; a hung child must not defeat recovery by continuing to write. -The capacity cost is bounded by the remaining recovery interval, the next acquisition attempt, and bounded process cleanup. Quiet crash leftovers wait the same interval as active ones; accept that small availability cost to avoid a second liveness test, report-age clock, or special quiet-run recovery procedure. The 15-minute margin is a proposed operating allowance, not a measured provider guarantee. Verify renewal and cleanup timing on private workers before relying on it. +The capacity cost is bounded by the remaining recovery interval, the next acquisition attempt, and bounded process cleanup. Quiet crash leftovers wait the same interval as active ones; accept that availability cost to avoid a second liveness test, report-age clock, or special quiet-run recovery procedure. Twenty minutes preserves the existing cold-profile operating allowance while removing its coupling to execution time; it is not a measured provider guarantee. Verify renewal and cleanup timing on private workers before relying on it. Two correctness limits remain. Failure to identify or stop a process must hold and report the worker rather than run tests over it; an unkillable process or broken provider needs operator repair, not another expiry that ignores the failed audit. Missing or corrupt **lock ownership** is also different from missing scratch metadata: dispatch cannot precede complete ownership registration, and an empty acquisition left before dispatch can be reclaimed after a clean worker audit, but ambiguous live work cannot be killed by a name guess. These limits prevent unsafe reuse; an ordinary crashed coordinator with attributable surviving processes follows automatic recovery and does not need an operator. @@ -94,11 +94,13 @@ Two correctness limits remain. Failure to identify or stop a process must hold a Older branches matter in two concrete ways. Their helpers still leave the two known directory prefixes, which every updated run removes. Their coordinators also unlock at shell exit and use timestamp expiry. New locks' `expiresAt: 0` stops the inspected old acquisition code from taking an updated run's worker, but cannot correct an old coordinator's own early release. Branches take the correction when they integrate main; updating every active branch or waiting for all old runs is not a prerequisite for this contribution or its final verification. -A legacy lock has no renewable deadline or complete outer-run attribution. Respect its unexpired TTL. After expiry, reclaim it only following a clean worker audit; keep and report one with live or ambiguous work that cannot be attributed safely. Do not infer an owner token from a process name or pretend an old coordinator supplies new renewal evidence. This limited legacy exception can cost capacity, but it avoids new mixed-version machinery. No permanent old-client exclusion mechanism or pool recreation is proposed. +A legacy lock has no renewable deadline or complete outer-run attribution. Respect its unexpired TTL. After expiry, reclaim it only following a clean worker audit; skip and report one with live or ambiguous work. The inspected old shard's exit trap removes only its own matching `lockId`, so ordinary old work frees itself when it ends. Do not infer an owner token from a process name or build a separate legacy recovery system. An old run that never exits remains an operator case; the pool deletion path below remains available. No permanent old-client exclusion mechanism or pool recreation is proposed. -`manage-blaxel-testshards.mjs` is the other actual API caller to correct. Its mutation commands must respect the same reservation: obtain locks on affected existing workers before deletion and refuse busy workers, releasing unused locks if acquisition fails. Reuse the common acquisition/recovery helper rather than inventing another expiry rule. Fresh provisioning runs with dispatch to that pool stopped until setup and architecture verification finish. Listing/status is read-only. This covers our pool script without adding a generic API gateway or maintenance framework; manual pool operations use the same drained-pool discipline. +`manage-blaxel-testshards.mjs` is the other actual API caller to correct. Deletion checks ownership under the same worker-local guard and refuses an unexpired reservation. After expiry, explicit pool deletion may reserve and delete even a worker whose process cleanup failed: destroying the sandbox is the cleanup. Remove that worker from test discovery before calling the delete API, so a failed or delayed deletion cannot admit a test run. Retain and report it as unavailable if deletion fails; do not silently return it to the pool. Missing ownership evidence cannot be read as an expired valid reservation. The manager continues to list such workers for operator repair. `ensure` remains fresh-only, so partial deletion does not refill capacity; the target pool must be deleted and recreated to replace missing workers. -Preserve the runner's read-only `--audit-stale-owners` mode: it must not recover another run or remove residue before observing it. It may reserve an available worker without recovery, holding that reservation through its existing dependency setup and report collection. A held lock is reported/skipped. This exception preserves the diagnostic's purpose; ordinary test acquisition and pool mutations use automatic recovery. +For fresh provisioning, publish the labels used by test-worker discovery only after runtime setup and architecture verification succeed. This closes the interval between sandbox creation and acquiring a lock without relying on a pool-wide dispatch stop. The manager must still identify and report partially provisioned workers, and must not silently reuse them through `createIfNotExists`. Listing/status is read-only. This covers our pool script without a generic API gateway or a new pool-draining protocol. + +Preserve the runner's read-only `--audit-stale-owners` mode: it must not recover another run or remove residue before observing it. It may reserve an available worker without recovery, holding that reservation through its existing dependency setup and report collection. A held lock is reported/skipped. This exception preserves the diagnostic's purpose; ordinary test acquisition uses automatic recovery, and pool deletion respects the same reservation deadline with the destruction exception above. ### A directory for each shard attempt @@ -143,7 +145,7 @@ The helper fix matters even with shard scratch: it releases files between test w **Convert five cases in two files, rather than the earlier 17-file set.** Josh delegated the scope judgment to this contribution. PR #32 (`982e564d`, merged 2026-10-09) removes conditional request headers at the registry Vite service boundary, so those services return full responses. Its recorded Chromium 145 evidence identifies the 2 MB buffer retained per 304 until context closure. That change directly addresses the repeated-reload mechanism for tests using those services, including tests that remain on Vite. Routine cleanup also recovers the leaked runtime capacity on each updated run. Neither mechanism now justifies a broad application-loader migration. -The two remaining source-loaded theme lifecycle cases are worth converting because they already claim built-application behavior in parts of their proofs and exercise repeated loads and server restarts. Browser demo mode is a small adjacent conversion: its three cases share one product-page helper, with repeated application reloads in two cases. Both files already have a real `ProductServer` and built assets available. This improves their shipped-asset boundary and removes hundreds of module requests per load without a new browser harness. The incremental memory and time benefit after #32 is unmeasured; the demo-mode flake is not proven to have been an OOM. +The two remaining source-loaded theme lifecycle cases are worth converting because they already claim built-application behavior in parts of their proofs and exercise repeated loads and server restarts. Browser demo mode is a small adjacent conversion: its three cases share one product-page helper, with repeated application reloads in two cases. Both files already have a real `ProductServer` and built assets available. This improves their shipped-asset boundary and removes hundreds of module requests per load without a new browser harness. The incremental memory and time benefit after #32 is unmeasured. Main's `f9180241` (PR #30) fixes the demo-mode flake by closing an unused HTTP connection that held fixture teardown open; that flake does not support the resource rationale for this conversion. Diminishing returns set in after these two files. The other product walks could gain packaged-asset coverage, but reviewing their loader, authentication, restart, and proof changes adds regression work without evidence of a remaining resource failure that those conversions would cure. Keep #32 for every remaining registry Vite service. Preserve the assertions and #9's merged theme-reset conversion. Validate the selected paths functionally and confirm their request inventory; any before/after resource comparison must use a baseline containing #32. This scope is based on source and proof inspection, not a post-#32 memory experiment. @@ -185,7 +187,7 @@ This contribution does not establish a post-#32 memory saving, eliminate unbundl ### Reconciliation with current main -PR #9 has merged. Integrate current main before deriving the affected spec/proof changes and implementation; preserve its theme-reset conversion and reconcile the other theme cases, authentication walks, and stable-front proxy with the actual merged API. The inventory above is explicitly tied to `b5477211`, not a claim that all listed lines remain on today's main. Do not convert an already converted case again. +Main at `a8e73fbf` is integrated in `74d22941`. Preserve PR #9's theme-reset conversion, the stable-front proxy's current API, SDK runtime assets, and single-origin application loading. The inventory above is explicitly tied to `b5477211`, not a claim that all listed lines remain on today's main. Do not convert an already converted case again. The five selected cases remain source-loaded at this checkpoint; proof descriptions will follow after independent spec review. Main also now owns full responses from registry Vite services and selection of a contributor's origin repository for remote checkout. Preserve both. Measure built-app memory savings against this new baseline rather than attributing PR #32's benefit to PR #12. @@ -199,7 +201,7 @@ Read kernel output through a bounded capture and retain at most **64 KiB of rele Periodic sampling, per-process descriptor inventories, cgroup probes, detailed browser/request event records, and browser-revision comparison remain optional investigation tools. They are not new merge obligations or spec requirements in this contribution. Do not add default videos, heap snapshots, or NetLogs. Existing native retry evidence and trace policy remain unchanged. -Write sidecars under the existing checked `.testshards/results` tree, outside disposable scratch, and download them with the other reports. Keep at most one attempt's bounded resource diagnostics per worker. A failed download leaves that attempt locked while the coordinator retries retrieval and renews; it does not start a growing queue of retained runs. Normally the next results reset follows download and release. If the coordinator stops renewing past the recovery deadline, automatic recovery may discard those abandoned reports and records that loss. Bound its recovery receipt to 128 KiB, with explicit truncation, within the incoming attempt's 8 MiB diagnostic budget. Coordinator-side `.test-runs/` retention is unchanged. No cross-run archive or retention daemon is needed. +Write sidecars under the existing checked `.testshards/results` tree, outside disposable scratch, and download them with the other reports. Keep at most one attempt's bounded resource diagnostics per worker. A failed download leaves that attempt locked while the coordinator retries retrieval and renews; it does not start a growing queue of retained runs. Normally the next results reset follows download and release. If the coordinator stops renewing past the recovery deadline, automatic recovery may discard those abandoned reports and records that loss. Its recovery receipt shares the incoming attempt's 8 MiB diagnostic budget, with explicit truncation; it needs no separate size limit. Coordinator-side `.test-runs/` retention is unchanged. No cross-run archive or retention daemon is needed. ## Authority and derivation @@ -220,8 +222,8 @@ All execution is on Blaxel at committed, pushed revisions. The shared developmen Derive proofs for the principal paths without a combinatorial fault matrix: - One run excludes a competing coordinator through preparation, teardown, and a deliberately delayed result download while renewing. Passing the former TTL never admits a second run beneath that valid reservation. Exercise both targeted and planned dispatch through the real provider. -- Stopping coordinator renewal permits automatic acquisition after the recovery interval, both with a quiet worker and with an owned supervisor/descendant still alive. Reuse follows confirmed termination; the receipt records any discarded reports. Contract coverage uses a controlled clock for renewal/deadline calculation, custom timeout, and error paths. A private-worker seam crosses real dispatch, recovery, and delayed retrieval, including a late dispatch rejected after ownership changes. It must not reduce to mocked API responses or simulate a coordinator crash by killing only the shard. -- Concurrent recovery attempts cannot both acquire; delayed renewal/release cannot alter a successor's reservation. Keep this at the lock helper's real filesystem boundary. The inspected older runner does not reclaim a new held lock with `expiresAt: 0`. Legacy helper residue is removed on the next updated run; ambiguous legacy ownership or failed process cleanup holds and reports. Pool-script mutation respects the same acquisition/recovery path. +- Stopping coordinator renewal permits automatic acquisition after the recovery interval, both with a quiet worker and with an owned supervisor/descendant still alive. Reuse follows confirmed termination; the receipt records any discarded reports. Contract coverage uses a controlled clock for renewal/deadline calculation, the recovery-interval override, and error paths. A private-worker seam crosses real dispatch, recovery, and delayed retrieval, including a late dispatch rejected after ownership changes. It must not reduce to mocked API responses or simulate a coordinator crash by killing only the shard. +- Concurrent recovery attempts cannot both acquire; delayed renewal/release cannot alter a successor's reservation. Keep this at the lock helper's real filesystem boundary. The inspected older runner does not reclaim a new held lock with `expiresAt: 0`. Legacy helper residue is removed on the next updated run; live legacy work is skipped and failed process cleanup prevents reuse. Pool deletion respects the same deadline but can destroy an abandoned worker that failed cleanup. Fresh workers are unavailable to tests until provisioning succeeds. - Exact-prefix and stale-scratch cleanup preserves unrelated files, live consumers, and the current attempt directory, while removing earlier scratch even when its metadata is incomplete. Real runtime builds demonstrate once-per-worker reuse, final disposal, and partial-build cleanup. A Blaxel observation verifies Node temporary paths and Chromium shared-memory descriptors under the attempt's `TMPDIR`, recording the actual browser version used. - A failed or recovered acceptance retains bounded start/end/OOM evidence through retrieval, including when its worker exits. Contract coverage handles unavailable probes, unchanged original test outcomes, truncation, and limits. @@ -237,12 +239,14 @@ Supply the coordinator's credentials through the Blaxel SDK's `process.exec` env The experiment plan adds at most three sandboxes. The spec's pool-size and tier examples are not a current capacity inventory: inspect actual workspace usage and provider limits before provisioning/dispatch. Do not claim slots are reserved or alter shared workers to obtain them. Delete this contribution's experiment workers through the pool script after retrieving their reports and before final standard 36-shard verify, leaving only its coordinator. That final run uses the normal `poc` acquisition path without intentional contamination or interruption; it does not require a pool-wide wait for other runs or a coordinated update of other branches. Archive its reports before disposing the coordinator. Record the exact tested revision, failures, recovered flakes, and any measurement limits. Ready-PR CI remains the repository's merge requirement; the contribution's execution evidence stays on Blaxel as instructed. -Keep browser-version alignment, shard count, browser request limits, retries, and production memory behavior outside this contribution. Forced GC is an investigation tool, not a proposed mitigation. The demo-mode and drag flakes have not been proven to share the resource-exhaustion cause. +Keep browser-version alignment, shard count, browser request limits, retries, and production memory behavior outside this contribution. Forced GC is an investigation tool, not a proposed mitigation. Preserve main's demo-mode teardown fix; the drag flake has no established resource-exhaustion cause. ## Review status Proposal round 1 at `1652787e`: FAIL on lease scope and mixed-version consequences. The response at `99b7ed93` narrowed the proposal while those human scope decisions were pending. Josh's 2026-10-27 instruction brings lock correction and recurring legacy cleanup into this PR and removes the pool-recreation choice. The [review response](test-harness-resource-cleanup-review-response.md) records the current dispositions without changing the independent review's historical verdict. -Proposal round 2 at `d3570bf4`: FAIL because dead-coordinator lock recovery is undefined. The [round 2 review](test-harness-resource-cleanup-review-r2.md) is preserved unchanged. Josh selected automatic recovery, required that an orphaned active run cannot hold a worker forever, and delegated the recovery mechanism and built-app conversion scope. His supervisor clarified that the suggested predicate and grace-period wording were illustrations, not requirements. This revision proposes one renewable recovery deadline, retains the ownership checks needed for safe reuse, narrows conversion to two files, and addresses refinements 2–4. It is ready for independent re-review, not converged. +Proposal round 2 at `d3570bf4`: FAIL because dead-coordinator lock recovery is undefined. The [round 2 review](test-harness-resource-cleanup-review-r2.md) is preserved unchanged. Josh selected automatic recovery, required that an orphaned active run cannot hold a worker forever, and delegated the recovery mechanism and built-app conversion scope. His supervisor clarified that the suggested predicate and grace-period wording were illustrations, not requirements. The revision at `57a701d0` supplied one renewable recovery deadline and narrowed conversion to two files. + +Proposal round 3 at `57a701d0`: **PASS**, relayed by the supervisor. The [review](test-harness-resource-cleanup-review-r3.md) is preserved unchanged. Its bounded refinements are applied here: fixed recovery interval, deletion of abandoned workers despite failed process cleanup, ordinary skipping of live legacy work, and trimmed receipt/provisioning language. The [response](test-harness-resource-cleanup-review-response.md#Round 3: convergence and spec derivation) records the judgments. The proposal is converged; the next checkpoint is independent review of the derived spec deltas. -No spec, proof, test, or implementation edits yet. No tests, builds, remote experiments, or worker operations were performed for this revision. Unknowns are live provider behavior during failed dispatch/recovery, today's worker browser versions, actual `TMPDIR` placement and post-mitigation memory savings, current capacity/credential availability, and observed recovery timing. The known cost is up to the configured worker timeout plus 15 minutes after the last renewal, then acquisition and cleanup; genuinely failed cleanup or unidentifiable legacy work still needs repair. Josh's calibration limits lock work to our runner and pool scripts. His spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. +No proof, test, or implementation work has begun beyond integrating main. No tests, builds, remote experiments, or worker operations were performed for this checkpoint. Unknowns are live provider behavior during failed dispatch/recovery, today's worker browser versions, actual `TMPDIR` placement and post-mitigation memory savings, current capacity/credential availability, and observed recovery timing. Recovery waits at most 20 minutes after the last renewal by default, then acquisition and bounded cleanup; failed cleanup prevents test reuse and explicit pool deletion remains available. Josh's spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. diff --git a/docs/working/test-harness-resource-cleanup-review-r3.md b/docs/working/test-harness-resource-cleanup-review-r3.md new file mode 100644 index 00000000..83a67eee --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-review-r3.md @@ -0,0 +1,70 @@ +# Proposal review, round 3: test-harness resource cleanup (draft PR #12, 57a701d0) + +**Verdict: PASS.** Round 2's blocker is resolved: the proposal now says what recovers a worker whose coordinator has gone, on what evidence, what happens to its reports, and what it costs, and the design follows Josh's decisions. The recovery design is close to as simple as it can safely be. Four refinements follow; the first two are worth doing before spec derivation, and none changes the verdict. + +- Reviewed: `docs/working/test-harness-resource-cleanup-proposal.md`, the review response and the copied round 2 review (identical to mine) at `57a701d0`, against Josh's round 2 decisions as the assignment and the response record them, the reviewer checklist, the code at the branch base `b5477211`, and current `origin/main` (`a8e73fbf`). +- I ran nothing. Every check is a read of files in the checkout or of git history. + +## The blocker is resolved + +| Round 2 asked for | The proposal now says | Checked | +| --- | --- | --- | +| An entry point | The canonical runner's `acquireAny()` recovers on finding an expired `recoverAfter`; the pool script's mutations reuse the same helper. | Consistent with `acquireAny` at `scripts/run-blaxel-testshards.mjs:333-358` (base), the only acquisition path. | +| Evidence that the coordinator is gone | The coordinator renews once a minute; the deadline is worker-local time of the last renewal plus the configured worker timeout plus 15 minutes; renewal cannot revive an expired reservation; only the coordinator renews. | Worker-local time avoids clock skew. Coordinator-only renewal is right: a shard that renewed for itself could hold a worker forever after its coordinator died. | +| A decision on undownloaded reports | Discarded when recovery runs, recorded in a receipt with the incoming attempt's setup sidecars; loss never becomes a passing result. | Resolves the "forever" reading of download-before-release. | +| The cost | 19 or 23 minutes after the last renewal, then one acquisition and bounded cleanup; quiet and active leftovers wait the same interval. | Sized and stated. | + +Code claims the design rests on, all verified: + +- A new lock written with `expiresAt: 0` is left alone by the old runner, which reclaims only a positive past timestamp (`run-blaxel-testshards.mjs:405-406`, base). +- An old shard's exit trap releases only a lock carrying its own `lockId` (`release_lock`, lines 773-775), so an old shard ending cannot remove a new owner's lock. The proposal could say this; it is why the legacy case is a skip, not a hazard (refinement 3). +- The owner token lives in one variable, `TV_TEST_SURFACE_OWNER`, and a surface supervisor replaces it for its children (`scripts/test/surface-supervisor.mjs:62`, main). A second attempt-level token that nested supervisors do not replace is therefore needed, as the proposal says. +- The existing reaper leaves processes with live supervisors untouched (`scripts/test/stale-owner-reaper.mjs:24, 96`). Recovery has to stop the abandoned run's live supervisor as well, which the proposal states without weakening the reaper for other owners. +- The worker timeout is the bound on the shard process itself: `process.exec({ timeout })` and `process.wait({ maxWait })` at lines 448 and 457. The spec's TTL and "final bound" sentences the proposal replaces are at `specs/arch/test-runner/blaxel-testshards.md:27-31`. + +## Is the recovery design as simple as it can safely be? Nearly + +What is simple and right: + +- One sequence for a quiet worker and for a still-active run, with no second liveness test and no report-age clock. That is the shape Josh asked for. +- One renewal, from the coordinator only, with the reason given. +- One guard (`flock` on a fixed file) for all worker-local lock changes. This is justified by a race in our own code: a renewal that read the lock before recovery replaced it would otherwise overwrite the new owner's record and fail the new run. It is one line of shell and is released by the kernel when a process dies. +- Orphaned reports are discarded, not archived; recovery is repeatable if it dies midway; a coordinator that finds its identity gone stops and reports. + +What can go without losing safety is in refinements 1 and 3. The one gap in the simple story is refinement 2. + +## Refinements (none blocking) + +1. **Use one fixed recovery interval, not "worker timeout plus 15 minutes".** Renewal is a timer, so the shard's execution bound has no bearing on how long a healthy coordinator can go without renewing: it renews every minute whatever the timeout. The timeout term adds a field recorded at acquisition, a custom-timeout case in the contract coverage, and two derived numbers to keep in step with the profiles. One interval does the same job. Reuse the existing `--lock-ttl-seconds` option (`blaxel-testshards.md:76`) as its override, which also lets the live recovery check on the private workers use a short interval instead of waiting at least 15 minutes per case. The renewal cadence is a free choice too: once a minute is 36 API calls a minute for a full run; a longer cadence with the same deadline costs nothing in safety. + +2. **The pool script must be able to delete the worker that needs repair.** The proposal has `delete` acquire each worker's lock and refuse busy ones, and names "operator repair" as the way out when a process cannot be stopped. Those two rules meet badly: the worker that cannot be cleaned is exactly the one the script would refuse to delete. Deleting a sandbox is the cleanup, so `delete` should refuse only a reservation that is still being renewed and proceed on one past its deadline, including one held because cleanup failed. Say also that a deleted worker is not re-added until the pool is recreated, since `ensure` is fresh-only (`scripts/manage-blaxel-testshards.mjs:41-47`); that is the status quo, and one sentence keeps an operator from expecting otherwise. + +3. **An expired legacy lock with live work is a skip, not a repair.** The proposal says to "keep and report" such a lock and, in its status section, that "unidentifiable legacy work still needs repair". Because the old shard's trap removes its own lock when it ends, an updated runner only has to skip that worker in this acquisition, as it already does for an unexpired lock; the worker frees itself. The only stuck case is an old shard that never exits, which is the status quo and ends when branches take main. Dropping the repair language removes a case from the design and from the spec. + +4. **Three sentences to trim or fix.** + - The 128 KiB cap on the recovery receipt. A receipt is a few hundred bytes of identities and outcomes, and the attempt's 8 MiB diagnostic budget already bounds it. + - "Fresh provisioning runs with dispatch to that pool stopped until setup and architecture verification finish." Nothing stops dispatch to a shared pool. If the intent is that `ensure` holds each new worker's lock until provisioning is verified, say so; otherwise rely on the architecture check that already skips an unprovisioned worker (`verifySandboxArchitecture`, lines 360-380, base) and drop the sentence. + - "Manual pool operations use the same drained-pool discipline." No such discipline is defined now that recreation is out of scope. + +## The rest of the proposal + +- **Built-application conversion.** Five cases in two files is a defensible judgment on the post-#32 evidence, and the proposal's facts hold on main: both files use a `ProductServer` through `appURL()` (`browser-demo-mode.test.ts:75-79`; `theme-product-acceptance.test.ts:694-695, 1065-1068`); the two theme cases restart the server (lines 845, 1785), and `restart()` boots a new port-0 backend (`test/helpers/product-server.ts:26-32`), so the stable front proxy is required for them, as the proposal says; `views/url-unsupported` is built (`packages/web/vite.config.ts:178`), served from Vite under `appURL()` (`shouldProxyToVite`, `product-server.ts:393-400`), and not copied by `installProductRuntimeAssets()` (lines 103-132), so the asset copy is a real prerequisite. The product proof calls the persistence walk "production shell modules through the test Vite server" at line 42 and the same walks "built application" at lines 44 and 51; the delivery proof does the same at lines 61 and 74-75. That is the description change the proposal anticipates. The testing directive is now scoped to `ProductServer` acceptances and excludes `appURLForServer()` walks, which settles round 2's refinement 1. +- **Round 2 refinements 2 to 4** are applied as proposed: every non-current attempt directory is removed after a clean audit; the old-branch preconditions are gone; the Chromium 148/145 discrepancy is stated and the build alias is still on main (`run-blaxel-testshards.mjs:923`); the credential path is supported by the code: the SDK reads `BL_API_KEY` and `BL_WORKSPACE` from the environment (`@blaxel/core` `common/settings.js:16-24`, `authentication/index.js:9-11`), `readBlaxelGithubToken()` reads the file and then `BLAXEL_TV_GH_TOKEN` (`scripts/test/blaxel-github-token.mjs:7-18`), and the runner passes only `GH_TOKEN` to workers (line 448). +- **Exposure table.** The three `appURL()` users found in round 2 are now listed; the reload in web `artifact-isolation.test.ts:149` and the second page in `artifact-move.test.ts:54` are as described. + +## Reviewer checklist + +- **Authority.** The derivation list names the right owners and says the Blaxel spec's TTL escape and shell-exit unlock are replaced. Add the fate of `--lock-ttl-seconds` (refinement 1) so the option table is not left describing a removed mechanism. +- **Testing directives.** The one directive is scoped and phrased as a default, not a prohibition. +- **Proportionality.** The lock and cleanup sections are long because Josh asked for those decisions here; the diagnostics commitment is the minimum; the conversion is two files. The length is in proportion to what is decided. +- **Status.** Accurate: ready for re-review, not converged; no spec, test or code edited; nothing run. + +## Evidence and limits + +- Read the proposal, the response and the copied review in full and diffed them against `d3570bf4`; read the base coordinator's lock, dispatch, wait, trap and timeout code, main's reaper, supervisor, token reader and pool script, the SDK's credential code, the two conversion targets, the three exposure-table files, both theme proofs, and the Blaxel spec's locking section. +- Not established: whether Blaxel's process `timeout` ends descendants (the proposal assumes not, which is the safe side); any rate limit on renewal calls; whether the provider can deliver a dispatch after a recovery interval (the shard-start lock check is cheap either way). + +## Housekeeping + +- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel. `git fetch origin` updated remote-tracking refs. +- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 4bf5f389..2f6e6d2b 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -1,6 +1,8 @@ # Resource-cleanup proposal: review response -Independent round 1 reviewed `1652787e`. Reviewer: Claude Fable (Nina), arranged by the supervisor. The [full review](test-harness-resource-cleanup-review-r1.md) remains unchanged: FAIL on two blockers. The response committed at `99b7ed93` narrowed the work pending Josh's decisions. This response records the revised proposal following his 2026-10-27 instruction; it does not claim reviewer convergence. +Independent round 3 reviewed `57a701d0`: **PASS**, relayed by the supervisor. The proposal has converged. The round 1 and 2 sections below retain the response history; [round 3](#Round 3: convergence and spec derivation) records the bounded refinements and current spec checkpoint. Reviewer: Claude Fable (Nina), arranged by the supervisor. + +Independent round 1 reviewed `1652787e`. The [full review](test-harness-resource-cleanup-review-r1.md) remains unchanged: FAIL on two blockers. The response committed at `99b7ed93` narrowed the work pending Josh's decisions. The following sections through round 2 record that history, not the current review verdict. ## Josh's scope decisions @@ -66,3 +68,32 @@ Refinement 1 is resolved by contributor judgment: **five cases in two files**, t | 4b: credentials for the Blaxel coordinator sandbox | Accepted. Specify process-environment injection of `BL_WORKSPACE`, an authorized `BL_API_KEY`, and `BLAXEL_TV_GH_TOKEN`, loaded in memory on the launching host. The installed SDK's authentication source and repository token reader support these paths. Do not copy credential files, embed values in commands/scripts/logs, or persist them in sandbox/Git configuration; verify availability before dispatch and end the credential-bearing process after report retrieval. No secret was read, copied, or transmitted while preparing this revision. | The revision is ready for independent re-review; the historical FAIL is unchanged until that review converges. No spec, proof, or implementation was edited. No test, build, browser/server launch, remote experiment, or worker operation ran. Live renewal/recovery behavior, browser versions, `TMPDIR` placement, and incremental resource benefit remain unmeasured; the proposal names the private Blaxel validation needed after spec/proof convergence. + +## Round 3: convergence and spec derivation + +The [round 3 report](test-harness-resource-cleanup-review-r3.md) is copied verbatim. It passes the proposal and raises no blocker. Under the workflow's bounded-refinement rule, apply the four substantive improvements without reopening cosmetic review or adding another proposal stage: + +| Refinement | Judgment and disposition | +| --- | --- | +| 1: fixed recovery interval | Accepted. Use 20 minutes, preserving the existing cold-profile allowance, independent of shard timeout. Reuse `--lock-ttl-seconds` for the interval override and derive renewal cadence from it. A healthy coordinator renews throughout ownership, so tying recovery to execution timeout bought no safety. Remove the custom-timeout field/case and the 19/23-minute distinction. This remains a proposed operating allowance to validate on Blaxel. | +| 2: delete a worker that cannot be cleaned | Accepted. Normal test acquisition still requires a clean audit. Explicit pool deletion refuses a valid reservation but may destroy an expired worker despite failed process cleanup. It first removes test-dispatch eligibility so a delayed provider deletion cannot hit a new test run. Failed deletion stays visible to pool management and unavailable to tests. Fresh-only `ensure` does not refill a partly deleted pool. This makes the stated operator repair path usable. | +| 3: live legacy work | Accepted. Skip/report it after TTL expiry and allow its matching-ID exit trap to free it normally. Do not call every live legacy run a repair case or add a second recovery mechanism. An old run that never exits still has the explicit pool deletion path. | +| 4: receipt and provisioning wording | Remove the separate receipt limit; the 8 MiB attempt budget already bounds it. Replace the undefined pool-draining language with delayed publication of test-discovery labels until runtime setup and architecture verification succeed. The existing architecture probe alone is insufficient to exclude a worker in the middle of setup once Node becomes available. Keep incomplete provisioning visible to the manager, with no silent reuse. No new pool-wide coordination protocol. | + +Main at `a8e73fbf` is integrated in `74d22941`, with the policy-level reconciliation in that merge message. Preserve the Vite full-response fix, PR #9's built reset, origin repository selection, single-origin app helper, SDK assets and current proxy API. PR #30 separately fixed demo-mode fixture teardown by closing idle HTTP connections; revise the resource proposal to say so. The five-case conversion is retained for its shipped-asset boundary and repeated loads, with no claim that it cures that flake or produces the historical memory saving. + +### Spec checkpoint + +| Owner | Derived delta | +| --- | --- | +| `specs/arch/testing-policy.md` | Worker-owned reusable builds with Playwright/Vitest teardown and failed-preparation cleanup; built application default limited to new/substantially changed `ProductServer` browser acceptances. | +| `specs/arch/test-runner/blaxel-testshards.md` | Exclusive renewable reservation through teardown and retrieval; fixed recovery interval and existing option; automatic owned-process cleanup and explicit abandoned-report loss; legacy skip; pool provisioning/deletion exclusion; per-attempt scratch and exact pre/post-run cleanup. Replace the TTL-only release promises rather than leaving contradictory authority. Dedicated fresh workers can prove provisioning without recreating the shared pool. | +| `specs/arch/test-runner/reporting.md` | Parent start/end resource observations, failed/recovered attempt retention, missing-evidence honesty, bounded kernel excerpts and sidecars, recovery receipts, and normalized links to retained diagnostics. | +| `specs/arch/test-runner/test-runner.md` | Cross-reference the Blaxel abandonment exception while preserving ordinary live-owner protection; preserve outer attempt attribution through nested scopes. | +| `specs/arch/test-runner/sharded-execution.md` | Carry the outer attempt environment and diagnostic artifact references across shared execution and provider normalization. No planning, shard-count, or retry change. | + +The rewritten spec policy on main governs this derivation. The proposal retains implementation rationale and the future experiment plan; specs own the lasting lifecycle decisions and cross-process/reporting contracts. Mirrored proofs and theme setup descriptions are intentionally unchanged at this checkpoint: they follow independent spec convergence. Product promises, production code, and the five conversion implementations are unchanged by this checkpoint. + +Validation is read-only source/policy review, exact-copy comparison of the independent review, and Git diff/whitespace inspection. No tests, builds, browser/server launches, remote experiments, or worker operations ran. Live recovery, temporary-file placement and bounded evidence transport remain to be proven on committed/pushed Blaxel revisions. The supervisor arranges independent spec review; Josh accepts the spec changes at PR time. + +Primitive surfaced the known recovery decisions and their later correction, alongside unrelated scope matches. Its discovery scan was truncated, so it was not a complete governing-set check. Repository policy and the supervisor's latest instructions governed the assessment; no decision was overridden or reconciled. diff --git a/specs/arch/test-runner/blaxel-testshards.md b/specs/arch/test-runner/blaxel-testshards.md index 94f05a97..4c983b16 100644 --- a/specs/arch/test-runner/blaxel-testshards.md +++ b/specs/arch/test-runner/blaxel-testshards.md @@ -1,10 +1,10 @@ -*The Blaxel shard coordinator: leasing a persistent sandbox pool, selecting the repository workers fetch, sourcing a GitHub token, preparing and locking workers, running shards with dependency-hash caching, and classifying Blaxel shard outcomes.* +*The Blaxel shard coordinator: persistent workers, exclusive leases and abandoned-run recovery, owned temporary files, remote checkout and execution, and retained shard results.* # Blaxel Test Shards Blaxel is the recommended way for agents and humans to run broad validation during development. It cuts the time a full run takes substantially — usually under three minutes, sometimes under two — which is the whole point. It works by keeping a pool of machines on hot standby, preconfigured so a run pays none of the usual setup cost: copying the container image, installing Playwright browsers, and the like. Routing runs to infrastructure like this, for speed and reliability, is also the principal reason Television has a single centralized test runner. -This spec is authoritative for the Blaxel coordinator: how it leases a persistent sandbox pool, selects the repository workers fetch and sources the GitHub token they fetch it with, prepares and locks each worker, runs the shard, caches dependencies by hash, classifies a Blaxel shard's outcome, and its options and output layout. The selected Node/npm values and `.nvmrc` authority are owned by [arch/node-versions.md](../node-versions.md). It extends the shared model in [sharded-execution.md](./sharded-execution.md); the git-safety and auth checks before dispatch are [preflight.md](./preflight.md); the canonical command surface that invokes it is [test-runner.md](./test-runner.md). +This spec is authoritative for the Blaxel coordinator: how it leases a persistent sandbox pool, recovers abandoned runs, owns temporary files, selects the repository workers fetch and sources the GitHub token they fetch it with, prepares and locks each worker, runs the shard, caches dependencies by hash, classifies a Blaxel shard's outcome, and its options and output layout. Resource evidence is owned by [reporting.md](./reporting.md#Resource diagnostics). The selected Node/npm values and `.nvmrc` authority are owned by [arch/node-versions.md](../node-versions.md). It extends the shared model in [sharded-execution.md](./sharded-execution.md); the git-safety and auth checks before dispatch are [preflight.md](./preflight.md); the canonical command surface that invokes it is [test-runner.md](./test-runner.md). Blaxel is the only remote provider. `npm test -- blaxel ` runs the coordinator `scripts/run-blaxel-testshards.mjs`; `npm test -- pool blaxel ` manages the pool through `scripts/manage-blaxel-testshards.mjs`. @@ -30,13 +30,54 @@ The pool manager `ensure` creates sandboxes named `tv-testshard---NN Provisioning requires an absent pool: delete the target pool before `ensure`, then let `ensure` create and provision every worker. Blaxel's `createIfNotExists` applies a creation specification only when a sandbox is absent; it does not reconcile provisioning commands or runtime state inside an existing sandbox. `ensure` therefore refuses a populated target pool, keeping provisioning fresh-only instead of carrying convergence logic for unknown nvm/Node state. +Fresh workers receive the labels that make them discoverable for test dispatch only after provisioning and architecture verification succeed. Pool management still identifies and reports partially provisioned workers; `ensure` must not treat them as absent or silently reuse them. Pool mutations follow [Locking](#Locking). A partly deleted pool is not refilled by `ensure`: replacing missing capacity requires deleting the remaining target pool and provisioning it afresh. + ## Locking -The coordinator leases one worker per shard through an in-sandbox lock directory `/tmp/tv-testshard.lock` with an `owner.json` recording `lockId`, `runId`, shard, owner, `expiresAt`, and `acquiredAt`. Normal runs use a bounded worker pool with at most 32 concurrent lease acquisitions (`MAX_PARALLEL_LEASE_ACQUISITIONS`); the pool starts another acquisition as each active attempt settles. Each attempt retains its unique shard lock ID, validates `uname -m` and `process.arch`, and records a lease as soon as it succeeds. The coordinator waits for every requested lease before starting any shard process. If pool discovery fails, the deployed pool is undersized, or acquisition is incomplete, no shard process starts. The attempt records one infrastructure-failed result per requested shard, including whether a lease was acquired, `executionStarted: false`, the assignment, and the exact discovery, acquisition, or barrier cause. Every successful partial lease is released, and the incomplete shards remain eligible for infrastructure retry. A lock whose `expiresAt` has passed may be reclaimed. +The coordinator leases one worker per shard through `/tmp/tv-testshard.lock`. Its `owner.json` records the unique `lockId`, run/shard/attempt identity, owner and acquisition time, worker-local last renewal and `recoverAfter` deadline, and dispatch ownership. Acquisition, renewal, dispatch registration, recovery, and release serialize their worker-local changes under one short OS-released guard outside the disposable lock directory. Rechecking identity under that guard prevents an earlier renewal or release from overwriting a new owner. The guard is not held during network downloads. ^blaxel-reservation + +Normal runs use at most 32 concurrent lease acquisitions (`MAX_PARALLEL_LEASE_ACQUISITIONS`), starting another as each settles. Each attempt retains its unique shard lock ID, validates `uname -m` and `process.arch`, and records a lease as soon as it succeeds. The coordinator waits for every requested lease before starting any shard process. Discovery failure, an undersized deployed pool, or incomplete acquisition starts no shard. The attempt records one infrastructure-failed result per requested shard, including whether a lease was acquired, `executionStarted: false`, the assignment, and the exact cause. Partial leases follow the release conditions below; incomplete shards remain eligible for infrastructure retry. + +Only the coordinator renews a reservation, throughout acquisition barriers, preparation, execution, teardown, and result download. Each successful renewal sets `recoverAfter` to worker-local time plus the recovery interval: 20 minutes by default, overridden by `--lock-ttl-seconds`. This interval is independent of the shard execution timeout. Renewal requires the same owner and an unexpired deadline; it cannot revive an expired reservation. A surviving shard cannot renew on its coordinator's behalf. + +The outer shard has a unique attempt owner token inherited by setup commands, surface supervisors, and their descendants. It remains available alongside the surface token that nested supervisors replace. Dispatch is recorded before the API request; the remote entry point checks the matching, unexpired reservation and registers its process name and PID/start identity under the guard before touching checkout or starting setup. A delayed request cannot launch obsolete work after recovery. Uploads use attempt-specific paths. Unknown dispatch outcomes retain the reservation until resolved or recovered. + +Normal release requires confirmed process teardown and completed retrieval of that attempt's reports. The shard shell finalizes evidence but never unlocks on exit. Named Blaxel process stop or successful shell completion does not establish that descendants are gone: audit the attributed processes and use token- and PID/start-validated TERM/KILL cleanup for survivors. Preserve the audit in the shard log. Remote cleanup activates nvm before enabling nounset, with strict handling in the cleanup body. An unreadable audit, uncertain dispatch, or surviving owner retains the reservation and excludes the worker from immediate test retries. ^blaxel-release + +While still renewing and holding the lock, download summaries, native reports, logs, and setup/cleanup/diagnostic artifacts; check their identities and required evidence. Missing or corrupt evidence remains an infrastructure failure. An early setup failure records which outputs were never produced and retains those that were. Incomplete retrieval keeps the reservation while retrieval is retried, without rerunning tests. An unused lease can be released only when execution was never requested, pending writes have settled, and any produced evidence has been retained. The fixed worker results tree must not be reset by a later checkout before this boundary. + +On `SIGINT` or `SIGTERM` while an attempt exists, the attempt becomes interrupted even if its shard promises have all just completed. Completed shard results remain intact. An unfinished shard retains downloaded summaries, native results, task artifacts, parsed task evidence, assignments, collections, and logs, and records `interrupted/coordinator-interruption` plus its known lease and execution state. A pending shard has no synthetic task result. A dispatched process whose start is unconfirmed omits `executionStarted`; absence means unknown, while `false` means execution was not requested. The coordinator makes this interrupted snapshot authoritative before cleanup and retrieval, and cleanup cannot replace it with an ordinary completion report. A report-write failure does not suppress process cleanup. The first signal exits 130 after cleanup; a second exits immediately. Any reservation that cannot be released safely stops renewing when the coordinator exits and follows abandoned-run recovery. A signal before the first attempt has no shard result to preserve; the canonical runner treats absent provider evidence as incomplete. Console summaries name only retained log files. + +### Abandoned-run recovery + +After renewal stops for the recovery interval, the next ordinary acquisition stops and audits the abandoned attempt's owned processes, clears its leftovers, and takes the lock. The same sequence applies to a quiet worker and to a surviving supervisor or test descendant. Expiry authorizes cleanup, never reuse beneath live processes. This operational definition of abandonment also covers a coordinator suspended or disconnected longer than the interval; it does not claim to prove the coordinator's OS process died. Quiet abandoned workers wait the same interval to keep recovery to one procedure. ^blaxel-recovery + +Under the guard, recheck the old identity and deadline, terminate only its attributed processes with revalidated identity, and confirm the worker has no remaining live or stale test owners. This recovery may stop the abandoned attempt's still-live supervisors; ordinary stale-owner cleanup continues to protect other live owners. Only then remove abandoned scratch and the known leftovers below, discard that attempt's undownloaded results, and publish the new reservation. A bounded [recovery receipt](./reporting.md#Resource diagnostics) identifies the old run, deadline, process cleanup, and discarded reports; it survives the results reset as part of the incoming attempt's evidence. This is the explicit exception to normal download-before-release. Lost evidence can never establish a pass for the abandoned run. + +Interrupted recovery leaves the old expired identity until another acquisition completes cleanup. A returning coordinator cannot renew, release, dispatch, or mutate an attempt whose ownership it lost; it reports infrastructure failure. Failure to identify or stop owners holds and reports the worker instead of dispatching tests. Missing or corrupt ownership evidence cannot authorize signalling by name. An acquisition interrupted before complete registration can be reclaimed after a clean worker audit, since execution cannot precede registration. Ordinary attributable coordinator crashes recover automatically; persistent cleanup/provider failures require operator action. -Ordinary attempt cleanup stops a dispatched remote process when completion was not confirmed, then runs the existing owner-lifecycle reaper while retaining the lease. Its remote shell activates nvm before enabling nounset because nvm activation is not nounset-safe; strict error, pipe, and unset-variable handling remains active for the reaper body. Named Blaxel process stop is not a process-tree guarantee: the reaper uses owner tokens and revalidated process identity/group signalling to remove detached descendants. Cleanup releases the lease only when the reaper confirms no live or stale owner-token processes remain; an unreadable, inconsistent, or contaminated report fails closed and leaves the exact lock to its TTL rather than exposing a dirty worker. The reaper report is appended to the shard log as lifecycle evidence. A process whose wait completed successfully is not stopped or reaped again, preserving the normal completed-process path; a lease whose execution was never requested is only released. This ordering prevents a timed-out or transport-failed process or descendant from continuing in an unlocked persistent sandbox while keeping completed and pre-dispatch cleanup unchanged. +### Older runners and pool operations -On `SIGINT` or `SIGTERM` while an attempt exists, the attempt becomes interrupted even if its shard promises have all just completed. Completed shard results remain intact. An unfinished shard retains downloaded summaries, native results, task artifacts, parsed task evidence, assignments, collections, and logs, and records `interrupted/coordinator-interruption` plus its known lease and execution state. A pending shard has no synthetic task result. A dispatched process whose start is unconfirmed omits `executionStarted`; absence means unknown, while `false` means execution was not requested. The coordinator makes this interrupted snapshot authoritative before cleanup, stops running shard processes, applies the same owner-confirmation boundary before releasing each known lease, and ensures cleanup cannot replace it with an ordinary completion report. A report-write failure does not suppress lease and process cleanup. A cleanup that cannot confirm zero owners retains its lock until TTL. The first signal exits 130 after cleanup; a second signal exits immediately, with lock expiry as the final bound. A signal handled before the first attempt has no shard result to preserve; the canonical runner treats the absent provider evidence as incomplete when it receives the coordinator exit. Console summaries name only retained log files. +Write `expiresAt: 0` in new records: older runners reclaim only positive past timestamps, so they leave these reservations alone. For a legacy record with no renewable deadline, respect its unexpired TTL; after expiry, require a clean worker audit before acquisition. Skip and report live or ambiguous legacy work. Its normal exit trap removes only its matching `lockId`. Updated runners cannot protect an older coordinator's downloads after that coordinator releases early; branches acquire the corrected behavior when they integrate it. No pool-wide drain or simultaneous branch update is required. + +Pool deletion checks the same identity and deadline under the guard and refuses unexpired reservations. An unlocked worker may be reserved for deletion after a clean owner audit. Explicit deletion may also reserve and destroy a worker with an expired reservation even when process cleanup failed, because sandbox deletion is the cleanup. Before issuing deletion, remove the worker from test dispatch eligibility; failed or delayed deletion must not admit a new test run. The manager continues to identify and report it as unavailable for operator repair. An unreadable ownership record is not proof of expiry. Read-only pool listing needs no reservation. Fresh provisioning follows [Pool](#Pool). + +`--audit-stale-owners` reserves an available worker without recovering a held reservation or deleting residue before observing it. It holds its reservation through its prerequisite setup and report collection, then releases normally. Held workers are reported/skipped. This preserves the diagnostic's read-only view of stale owners. + +## Temporary files + +Every leased shard attempt has a fresh private directory under `/tmp/tv-testshards/`, identified with its run, shard, attempt, and lock. An absolute `TMPDIR` points to its `tmp/` child before preparation or tests begin and is preserved by child environments. Uploaded scripts, plans, askpass helpers, and setup sidecars use the attempt directory; finalized reports live outside it. Paths stay short enough for browser Unix sockets. Intentional npm, Electron, browser-download, and dependency-hash caches retain their existing persistent locations. ^blaxel-scratch + +After taking the reservation and confirming no prior execution remains, each run removes only: + +- Real directories directly under `/tmp` matching `television-source-cli-e2e-*` or `television-web-bundle-*`. +- Every non-current attempt directory directly under `/tmp/tv-testshards/`, including ones with missing or incomplete metadata. + +The lock and clean worker audit establish disuse; metadata or age alone does not. Prior result retrieval is protected by the reservation, with only the abandoned-run exception above. Reject symlink roots, unexpected mount boundaries, and paths escaping the allowed parent; recursive removal must not follow links into another tree. Preserve the current attempt, lock and guard, provider uploads, X11 state, unrelated temporary files, checkouts, and persistent caches. Report counts and allocated bytes removed per prefix, filesystem free space before/after, and skips/errors. Do not equate allocated bytes removed with free-space change or claim a partial deletion reclaimed an entire directory. ^blaxel-known-leftovers + +After success, failure, timeout, or interruption, stop attributed processes, finalize evidence outside scratch, and remove the current attempt directory while retaining the lock through download. An uncertain teardown preserves scratch. Failed cleanup retains its cause and native outcomes, but prevents new test dispatch until cleanup succeeds; an inconclusive worker audit likewise skips deletion and dispatch. Download available evidence even when cleanup fails. Directory deletion cannot release Chromium's open, unlinked shared-memory files: their owning processes must stop first. + +Retain at most the current attempt's bounded resource evidence on the worker under the existing `.testshards/results` tree. The next reset follows protected download, or the explicitly recorded abandonment decision above. There is no accumulating cross-run diagnostic archive. Coordinator-side run retention is unchanged. Evidence contents and byte limits belong to [reporting.md](./reporting.md#Resource diagnostics). ## Shard lifecycle @@ -69,7 +110,7 @@ The coordinator attributes failures through remote wait and final log capture to ## Options -The coordinator's option set, with code-true defaults. The canonical runner ([test-runner.md](./test-runner.md)) overrides some of these, noted inline. +The coordinator's options and defaults follow. The canonical runner ([test-runner.md](./test-runner.md)) overrides some of these, noted inline. | Option | Default | Notes | |---|---|---| @@ -81,16 +122,17 @@ The coordinator's option set, with code-true defaults. The canonical runner ([te | `--arch` | `x64` | other values rejected. | | `--commit` | `HEAD` | the committed revision to test. | | `--timeout-profile` | `normal` | the canonical runner passes `cold`. See drift note below. | -| `--worker-timeout-seconds`, `--lock-ttl-seconds` | per profile | override the profile values. | +| `--worker-timeout-seconds` | per profile | override the shard execution timeout. | +| `--lock-ttl-seconds` | `1200` | positive recovery interval since the last successful renewal, independent of timeout profile; expiry requires cleanup before test reuse. | | `--retry-infra` | `0` | the canonical runner passes `2`. | | `--allow-dirty` | off | dispatch from a dirty tree, ignoring local edits. The canonical runner's `--ignore-uncommitted` is translated to this. | | `--test-retries` | `0` | non-unit retry count passed to the worker. | | `--repo-url` | the checkout's `origin` as a GitHub HTTPS URL | repository fetched in the sandbox ([#Repository](#Repository)). | | `--playwright-browsers-path` | `/home/playwright/.cache/ms-playwright` | browser cache path. | | `--output-dir` | `.blaxel-testshards/latest` | run output location. | -| `--audit-stale-owners` | off | lease one worker, install/check system prerequisites, run the read-only stale-owner diagnostic, print its JSON result, and release the lease without dispatching tests. It may be combined only with `--pool` and `--arch`. | +| `--audit-stale-owners` | off | reserve one available worker without recovery or residue removal, install/check system prerequisites, run the read-only stale-owner diagnostic, print its JSON result, and release without dispatching tests. It may be combined only with `--pool` and `--arch`. | -Timeout profiles (code-true): `normal` = worker 240 s, lock TTL 900 s; `cold` = worker 480 s, lock TTL 1200 s; `repair` = worker 480 s, lock TTL 1200 s. +Worker execution timeout profiles: `normal` = 240 s; `cold` = 480 s; `repair` = 480 s. All profiles use the same default recovery interval above. **Default profile.** The coordinator's own default timeout profile is `normal`, but the canonical runner invokes Blaxel with `--timeout-profile cold`, so a `npm test -- blaxel` run uses `cold` unless overridden. @@ -125,7 +167,8 @@ Under the testing policy's [rule for integration against paid services or comple `--suite target` is not sufficient for a change that affects plan transfer; [sharded-execution.md's Testing section](./sharded-execution.md#Testing) defines the required evidence. -A change to pool provisioning or runtime selection requires the live run to use freshly recreated `poc` workers, since `ensure` refuses a populated pool ([#Pool](#Pool)). +A change to pool provisioning or runtime selection requires the live run to use freshly provisioned workers, since `ensure` refuses a populated pool ([#Pool](#Pool)). A dedicated pool may supply this evidence without deleting shared `poc` workers. -A change to repository selection, or to how a worker's checkout moves between repositories, requires live runs from checkouts of two repositories on the same pool, including a run on workers whose checkout last served the other repository. +Recovery and cleanup experiments use disposable owned processes on a dedicated pool, never intentionally contaminating shared workers. Existing `isolated-github-only` fixtures retain their [placement restriction](./test-runner.md#Destructive fixture placement). Temporary-file placement needs live observation of Node-created paths and Chromium shared-memory descriptor paths under the attempt's `TMPDIR`, with the actual launched browser version and executable recorded; the Playwright package version alone does not establish that browser identity. +A change to repository selection, or to how a worker's checkout moves between repositories, requires live runs from checkouts of two repositories on the same pool, including a run on workers whose checkout last served the other repository. diff --git a/specs/arch/test-runner/reporting.md b/specs/arch/test-runner/reporting.md index 0cc4fab1..a0eea8ad 100644 --- a/specs/arch/test-runner/reporting.md +++ b/specs/arch/test-runner/reporting.md @@ -1,10 +1,10 @@ -*The run directory, normalized reports, timing events, and CI run artifacts every test run produces.* +*The run directory, normalized reports, resource diagnostics, timing events, and CI run artifacts test runs produce.* # Reporting Every test run leaves a compact result for people and agents, detailed files for debugging, and machine-readable timing records. The timing records that Blaxel runs leave in their run directories are the input for balancing future test runs across parallel machines. -This spec is authoritative for the *run directory*, the *normalized surface result* model, the per-run timing event stream, compact-summary discipline, status accounting, and the run artifacts GitHub Actions jobs upload. Local runs, regular GitHub CI jobs, and Blaxel runs all produce this model; Blaxel provider results are translated into it by [sharded-execution.md](./sharded-execution.md). The runner that drives reporting is [test-runner.md](./test-runner.md); the testing discipline behind the honesty of these reports is [testing-policy.md](../testing-policy.md). Duration rollups and shard planning are owned by [sharded-execution.md](./sharded-execution.md). +This spec is authoritative for the *run directory*, the *normalized surface result* model, Blaxel resource diagnostics, the per-run timing event stream, compact-summary discipline, status accounting, and the run artifacts GitHub Actions jobs upload. Local runs, regular GitHub CI jobs, and Blaxel runs all produce the shared reporting model; Blaxel provider results are translated into it by [sharded-execution.md](./sharded-execution.md). The runner that drives reporting is [test-runner.md](./test-runner.md); the testing discipline behind the honesty of these reports is [testing-policy.md](../testing-policy.md). Duration rollups and shard planning are owned by [sharded-execution.md](./sharded-execution.md). ## Run directory @@ -82,6 +82,7 @@ interface NormalizedSurfaceResult { logPath: string | null; nativeResultPath?: string | null; nativeResultPaths?: string[]; + resourceDiagnosticsPaths?: string[]; counts: Counts; failedTests: Array>; flakyRecoveredTests: Array>; @@ -210,6 +211,20 @@ A process leak makes every surface named by `owningSurfaceIds` failed with `infr `events.ndjson` is an explicit exception to compact-summary discipline. It enumerates every test attempt because its purpose is machine analysis; it is not printed into agent context as a normal command result. +## Resource diagnostics + +Each Blaxel surface execution, planned or targeted, records available RAM, shared-memory usage, free bytes on the effective temporary filesystem, and the Linux OOM-kill counter at start and end. The parent captures and immediately retains the start observation before surface preparation, then captures the end after teardown, so a browser or test-worker crash does not erase the baseline. Each value records its units and observation time; unavailable probes and missing final observations are explicit, never substituted with zero. Counter resets cannot be reported as negative OOM events. ^resource-diagnostics + +One sidecar identifies the run, shard attempt, and surface execution and links to its native attempt evidence. Retain it for failed tests, failures that recover on retry, and missing native output. The surface result's `resourceDiagnosticsPaths` links only retained files relative to the owning run directory. Attempt history keeps its own sidecars when an infrastructure retry supersedes that attempt; aggregation does not overwrite them with the successful retry's evidence. A sidecar with missing native evidence does not invent a test result or a selected-surface outcome. + +If any native attempt failed, or execution evidence is incomplete, capture relevant kernel OOM lines for the execution interval through a bounded read. Keep at most 64 KiB of these lines per surface execution, 128 KiB for the entire sidecar, and 8 MiB of resource diagnostics per shard attempt, including recovery receipts. Reserve space for compact start/end records before optional excerpts. State truncation or omission and unavailable-probe reasons. Probe or diagnostic-write failure adds an observability warning without changing the test outcome or masking its original failure. ^resource-diagnostic-bounds + +Start/end observations give context and an OOM-counter delta, not a peak-memory measurement or the exact failing allocation. A system counter increase alone does not identify which process was killed. Kernel evidence may identify the process; absent or inaccessible kernel output is unknown, not evidence of no OOM. No periodic sampler, full process inventory, heap snapshot, video, or NetLog is required by this contract. + +A recovery receipt records the prior worker/run/lock identity, last renewal and deadline, process identities stopped and cleanup outcome, and whether undownloaded reports were discarded. Preserve it with the incoming attempt's setup evidence across results reset and download, within the same attempt diagnostic budget. A receipt disclosing discarded reports supplies no passing evidence for that abandoned run. [Blaxel's lifecycle](./blaxel-testshards.md#Abandoned-run recovery) owns when such discard is permitted and how worker retention stays bounded. + +These artifacts contain no credentials, request query strings or bodies, or arbitrary process environments; process identities exclude arguments. They remain outside disposable scratch and are downloaded with the provider artifacts. Resource samples and kernel excerpts stay out of the timing event stream and compact console output; links make the detailed evidence available. Existing native retry records and trace policy remain unchanged. + ## Timing event stream `events.ndjson` is UTF-8 newline-delimited JSON. Each event is serialized with RFC 8785 JSON canonicalization followed by one LF byte; the file also ends in LF. Each line is one `TimingEvent`. The stream is complete only after the run has finished; it is produced from native reports and provider output, not treated as a live progress protocol. diff --git a/specs/arch/test-runner/sharded-execution.md b/specs/arch/test-runner/sharded-execution.md index 07f702b8..8657c4ec 100644 --- a/specs/arch/test-runner/sharded-execution.md +++ b/specs/arch/test-runner/sharded-execution.md @@ -225,6 +225,8 @@ Blaxel workers and regular GitHub CI matrix jobs run `scripts/run-test-shard.mjs - `--fail-fast` stops the shard after the first failed group or task. Every later task that had files assigned by the plan remains in the shard summary with `status: "skipped"`, `skipReason: "fail-fast"`, its assigned files, and no collected files or runner artifacts. - The worker writes `shard-.json` into its results directory, default `.testshards/results`, and exits non-zero if any task failed or plan validation failed. +On Blaxel, the shared worker preserves the outer attempt's ownership and temporary-directory environment; [the Blaxel lifecycle](./blaxel-testshards.md#Locking) owns these, including teardown and download before release. Resource sidecars travel with the task artifacts under [reporting's resource-diagnostics contract](./reporting.md#Resource diagnostics), including evidence from superseded infrastructure attempts. GitHub workers retain their existing host lifecycle. + ## Blaxel provider report shape The Blaxel coordinator collects the plan and per-shard summaries into a provider report. Regular GitHub CI retains each direct worker's shard summary and timing stream as job artifacts; the required agreement join is owned by [github-ci.md](./github-ci.md) under the invariants above. @@ -250,6 +252,7 @@ interface ProviderTask { failureStep?: string | null; processLeaks?: NormalizedProcessLeak[]; nativeResultPath?: string | null; + resourceDiagnosticsPaths?: string[]; [key: string]: unknown; } @@ -338,4 +341,3 @@ A seam must also pass the generated `--test-list` file through the real Electron Under the testing policy's [rule for integration against paid services or complex host dependencies](../testing-policy.md#integration-against-paid-services-or-complex-host-dependencies-is-important), a live planned Blaxel suite run must cross the real Blaxel API and the real transport to the remote process. The run must show that the worker validates and executes the `ShardPlan` it receives. A `--suite target` run carries no provider plan, so it cannot prove this handoff. Live GitHub evidence follows the [isolated CI requirements in github-ci.md](./github-ci.md#Testing). The evidence must show that independently started matrix jobs derive the same valid plan from their shared inputs. The strict join must validate the jobs' summaries and exact file assignments. - diff --git a/specs/arch/test-runner/test-runner.md b/specs/arch/test-runner/test-runner.md index 3a376055..3a6e584a 100644 --- a/specs/arch/test-runner/test-runner.md +++ b/specs/arch/test-runner/test-runner.md @@ -219,6 +219,8 @@ A combined Vitest run uses a generated config with one `test.projects` entry per Persistent workers inspect Television owner tokens before accepting a new scope. A token is stale only when its recorded supervisor PID and process start time no longer identify a live supervisor; a concurrent run with a live supervisor is left untouched. A stale process is captured and reaped, and the worker reports an infrastructure preflight failure naming the token and process; it does not run a new surface in contaminated state. An ephemeral provider still performs the check so its correctness does not depend on host disposal. +Blaxel additionally reserves the whole worker through result retrieval. Its [abandoned-run recovery](./blaxel-testshards.md#Abandoned-run recovery) may stop that abandoned attempt's live supervisors after coordinator renewal expires; it does not weaken the ordinary stale-owner rule for other runs. Surface supervisors and helpers preserve the outer attempt attribution alongside their own surface tokens. [Reporting](./reporting.md#Resource diagnostics) owns the resource evidence captured across Blaxel surface preparation, execution, and teardown. + ## End-of-surface leak detection After registered teardown and its graceful interval, the scope owner enumerates every live process carrying the owner token. Each candidate is revalidated immediately before cleanup. A candidate that exits naturally before any cleanup signal is sent is not a *process leak* and is omitted; this closes the normal child-exit race where a runner exits just before one of its helper processes. A process that requires `SIGTERM` or `SIGKILL` remains a leak even if a later revalidation finds that it has exited. For every actual leak, the owner captures its identity and listening sockets before sending `SIGTERM`, escalates to `SIGKILL` after the bounded interval, and scans once more to confirm that no survivor remains. Before each signal it confirms that the PID still has the same process start time and owner token, preventing PID reuse from targeting another process. The diagnostic identifies at least the surface owner, PID, parent PID, process group, process start time, executable command, and every attributable listening TCP socket. [reporting.md](./reporting.md) owns the normalized record that carries this attribution. ^test-process-leak @@ -258,4 +260,3 @@ Under the testing policy's [mocking policy](../testing-policy.md#Mocking policy) Under the testing policy's [compositional coverage rule](../testing-policy.md#Compositional coverage across clean seams), this spec owns the decision that selects a provider automatically from the home marker and, on marked hosts, a remote preflight result. [Preflight's Testing section](./preflight.md#Testing) requires a real Git process to run the production git-safety sequence in a repository with an `origin` remote. It also requires live evidence for the Blaxel capability checks. The live planned run required by [sharded execution's Testing section](./sharded-execution.md#Testing) supplies the real provider report for the remote run. Reading `.github/workflows/ci.yml` proves only which command the file contains. The isolated live run required by [GitHub CI's Testing section](./github-ci.md#Testing) establishes that GitHub starts the job and that the job invokes the test runner. - diff --git a/specs/arch/testing-policy.md b/specs/arch/testing-policy.md index 9d4f5086..8319400f 100644 --- a/specs/arch/testing-policy.md +++ b/specs/arch/testing-policy.md @@ -207,6 +207,10 @@ In order to liberally cover boundary seams across the codebase, complex harness Prefer real, committed fixtures over invented data. Better yet, generate fixtures dynamically from current code paths, driven by simpler inputs with fewer assumptions, whenever possible. +Reusable temporary builds belong to the test worker that creates them. The source-CLI runtime is built once per worker and versioned web bundles once per version per worker; test-case cleanup stops that case's servers without discarding these shared builds. Worker teardown waits for pending builds and stops their consumers before removing the outputs, in both Playwright and Vitest. Failed preparation removes its partial runtime and leaves the cache retryable; failed launch preparation also removes its per-launch copy and home. A cached path is published only after preparation succeeds. Cleanup is idempotent and never removes files beneath a surviving consumer. The [Blaxel attempt lifecycle](test-runner/blaxel-testshards.md#Temporary files) supplies the backstop for a killed worker that cannot run its teardown. ^worker-build-lifetime + +For new or substantially changed browser acceptances that use the `ProductServer` helper, the default is its built application unless the assertion needs development-module behavior. This exercises shipped assets and avoids loading hundreds of development modules for a product assertion. This default does not require retroactive conversion of other tests or apply to in-process `Server` walks using `appURLForServer()`, component fixtures, or Electron module/dispatcher seams. ^product-acceptance-built-app + ## Timing, observable status, cleanup, and flakiness Prefer real readiness and state observation over sleeps. Take the time to make complex tasks reliably observable for testing purposes. ^flake-avoidance diff --git a/specs/index.md b/specs/index.md index 2e5833ec..70ca30cd 100644 --- a/specs/index.md +++ b/specs/index.md @@ -40,7 +40,7 @@ Generated from the first-line description of every spec by `scripts/specs-index. - `licensing.md` (294 lines) — *The licensing implementation: bundle-derived third-party notices generation for the esbuild, Vite, and skill outputs, the config file for elections, notice texts, and ignored packages, the vendored-asset provenance manifest, the committed source-surface notices, the suite-run allowlist gate, and the assertions on the CLI tarball and the desktop upload directory — wired into the existing build, verify, and publish pipeline.* - `making-skills.md` (30 lines) — *Arch spec: how a bundled skill gets made — authority, bundle derivation, and final consumer delivery.* - `node-versions.md` (68 lines) — *The Node and npm versions used for repository work, automation, publishing, and published-package support.* - - `testing-policy.md` (234 lines) — *How Television is tested: coverage requirements, the fixture/mock distinction, the three test shapes and their declaration schema, and mock vs integration discipline.* + - `testing-policy.md` (238 lines) — *How Television is tested: coverage requirements, the fixture/mock distinction, the three test shapes and their declaration schema, and mock vs integration discipline.* - **artifact-frame/** - `artifact-bridge.md` (354 lines) — *The artifact bridge: how an embedded artifact document and the app cooperate across the iframe/webview boundary — lifecycle and readiness, navigation reporting, input observation, keyboard forwarding, and live updates for shared artifacts.* - `index.md` (51 lines) — *The artifact-frame architecture root: the domain's module map, the frame lifecycle rules, and the deliberate boundary between what is specified here and what remains governed by code.* @@ -90,14 +90,14 @@ Generated from the first-line description of every spec by `scripts/specs-index. - `sink.md` (63 lines) — *The telemetry sink: the PostHog transport, its bounded in-memory buffer, fire-and-forget delivery that never blocks or crashes the server, and the platform operations (region, key, agent access).* - **test-runner/** - `attestation.md` (132 lines) — *Tree-hash test attestation: recording that an exact tree passed full validation as a bare `refs/testpass//` ref, and letting validation-bearing CI runs skip their heavy test jobs — never their cheap ones — when the identical tree is already attested under the current version.* - - `blaxel-testshards.md` (131 lines) — *The Blaxel shard coordinator: leasing a persistent sandbox pool, selecting the repository workers fetch, sourcing a GitHub token, preparing and locking workers, running shards with dependency-hash caching, and classifying Blaxel shard outcomes.* + - `blaxel-testshards.md` (174 lines) — *The Blaxel shard coordinator: persistent workers, exclusive leases and abandoned-run recovery, owned temporary files, remote checkout and execution, and retained shard results.* - `flaky-tests.md` (74 lines) — *The flaky-test policy: the runner's retry budget, the per-test `FLAKY_TEST_RETRIES` annotation, and how recovered flakes are surfaced.* - `github-ci.md` (137 lines) — *The GitHub Actions CI execution contract: PR concurrency, per-job dependency and browser environment, build-once fan-out with an artifact manifest, cleanup tool prerequisites, worker counts, and the retirement record for the Blacksmith-dispatched shard provider.* - `preflight.md` (92 lines) — *The checks the test runner runs before tests: local capability checks for the selected surfaces, and the git-safety and provider-auth checks that gate a committed-revision remote run.* - - `reporting.md` (411 lines) — *The run directory, normalized reports, timing events, and CI run artifacts every test run produces.* - - `sharded-execution.md` (341 lines) — *The shared remote execution model: deterministic duration-aware plans, per-worker execution, provider reports, retries, and normalization.* + - `reporting.md` (426 lines) — *The run directory, normalized reports, resource diagnostics, timing events, and CI run artifacts test runs produce.* + - `sharded-execution.md` (343 lines) — *The shared remote execution model: deterministic duration-aware plans, per-worker execution, provider reports, retries, and normalization.* - `test-registry.md` (181 lines) — *The test registry: how `test.config.mjs` declares the surfaces, suites, and execution groups the test runner selects, owns files against, and validates.* - - `test-runner.md` (261 lines) — *The canonical `npm test` and `npm run verify` command surface: how a caller selects tests, picks where they run, and runs them, with the guardrails that keep broad runs honest.* + - `test-runner.md` (262 lines) — *The canonical `npm test` and `npm run verify` command surface: how a caller selects tests, picks where they run, and runs them, with the guardrails that keep broad runs honest.* - **themes/** - `authoring.md` (204 lines) — *Theme-authoring guidance: the self-contained theming document bundled inside the `television` skill, the authoritative UI material it carries, and the workflow it gives an agent.* - `bundled-installation.md` (77 lines) — *Bundled theme installation: validated package assets, CLI path resolution, minimum-version installation and replacement with backup copies, and the one-time default-theme selection.* From 9fd51f490f7227ccf26f2e45821c6777049c985c Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:26:38 +0000 Subject: [PATCH 06/64] Derive resource cleanup proofs and refine lease specifications --- .../test-harness-resource-cleanup-proposal.md | 10 +-- ...arness-resource-cleanup-review-response.md | 35 +++++++++- ...harness-resource-cleanup-spec-review-r1.md | 65 +++++++++++++++++++ proofs/arch/test-runner/blaxel-testshards.md | 44 +++++++++++-- proofs/arch/test-runner/reporting.md | 17 ++++- proofs/arch/test-runner/sharded-execution.md | 3 +- proofs/arch/test-runner/test-runner.md | 2 + proofs/arch/testing-policy.md | 24 ++++++- proofs/arch/themes/delivery.md | 4 +- proofs/index.md | 14 ++-- proofs/product/artifacts.md | 6 +- proofs/product/themes-and-appearance.md | 6 +- specs/arch/test-runner/blaxel-testshards.md | 44 ++++++------- specs/arch/test-runner/sharded-execution.md | 2 +- specs/arch/test-runner/test-runner.md | 2 +- specs/arch/testing-policy.md | 2 +- 16 files changed, 228 insertions(+), 52 deletions(-) create mode 100644 docs/working/test-harness-resource-cleanup-spec-review-r1.md diff --git a/docs/working/test-harness-resource-cleanup-proposal.md b/docs/working/test-harness-resource-cleanup-proposal.md index 621ef5c9..fde4e64e 100644 --- a/docs/working/test-harness-resource-cleanup-proposal.md +++ b/docs/working/test-harness-resource-cleanup-proposal.md @@ -1,6 +1,6 @@ # Test-harness resource cleanup -Converged proposal, revised 2026-10-27 with round 3's bounded refinements. Owner: rose. Branch: `thopter/test-harness-resource-cleanup`, draft PR #12, originally based on `b5477211`. Main at `a8e73fbf` was integrated in `74d22941` before spec derivation. The supervisor arranges independent review of the proposal, specs, proofs, and implementation; Josh reviews every spec delta at PR time. No production behavior changes are proposed. +Converged proposal, revised 2026-10-27 with bounded proposal/spec review refinements. Owner: rose. Branch: `thopter/test-harness-resource-cleanup`, draft PR #12, originally based on `b5477211`. Main at `a8e73fbf` was integrated in `74d22941` before spec derivation. The supervisor arranges independent review of the proposal, specs, proofs, and implementation; Josh reviews every spec delta at PR time. No production behavior changes are proposed. ## Problem and intended result @@ -96,7 +96,7 @@ Older branches matter in two concrete ways. Their helpers still leave the two kn A legacy lock has no renewable deadline or complete outer-run attribution. Respect its unexpired TTL. After expiry, reclaim it only following a clean worker audit; skip and report one with live or ambiguous work. The inspected old shard's exit trap removes only its own matching `lockId`, so ordinary old work frees itself when it ends. Do not infer an owner token from a process name or build a separate legacy recovery system. An old run that never exits remains an operator case; the pool deletion path below remains available. No permanent old-client exclusion mechanism or pool recreation is proposed. -`manage-blaxel-testshards.mjs` is the other actual API caller to correct. Deletion checks ownership under the same worker-local guard and refuses an unexpired reservation. After expiry, explicit pool deletion may reserve and delete even a worker whose process cleanup failed: destroying the sandbox is the cleanup. Remove that worker from test discovery before calling the delete API, so a failed or delayed deletion cannot admit a test run. Retain and report it as unavailable if deletion fails; do not silently return it to the pool. Missing ownership evidence cannot be read as an expired valid reservation. The manager continues to list such workers for operator repair. `ensure` remains fresh-only, so partial deletion does not refill capacity; the target pool must be deleted and recreated to replace missing workers. +`manage-blaxel-testshards.mjs` is the other actual API caller to correct. Deletion checks ownership under the same worker-local guard and refuses an unexpired reservation. After expiry, explicit pool deletion may reserve and delete even a worker whose process cleanup failed: destroying the sandbox is the cleanup. Remove that worker from test discovery before calling the delete API, so a failed or delayed deletion cannot admit a test run. Retain and report it as unavailable if deletion fails; do not silently return it to the pool. Unreadable or incomplete records follow one rule for acquisition and deletion: reclaim only after a clean worker audit, otherwise hold and report. A fresh worker never published for test dispatch may be deleted without a lease. Any other worker whose ownership check cannot run is held for operator repair; current discovery ineligibility alone does not prove an earlier run stopped. The manager continues to list such workers for operator repair. `ensure` remains fresh-only, so partial deletion does not refill capacity; the target pool must be deleted and recreated to replace missing workers. For fresh provisioning, publish the labels used by test-worker discovery only after runtime setup and architecture verification succeed. This closes the interval between sandbox creation and acquiring a lock without relying on a pool-wide dispatch stop. The manager must still identify and report partially provisioned workers, and must not silently reuse them through `createIfNotExists`. Listing/status is read-only. This covers our pool script without a generic API gateway or a new pool-draining protocol. @@ -187,7 +187,7 @@ This contribution does not establish a post-#32 memory saving, eliminate unbundl ### Reconciliation with current main -Main at `a8e73fbf` is integrated in `74d22941`. Preserve PR #9's theme-reset conversion, the stable-front proxy's current API, SDK runtime assets, and single-origin application loading. The inventory above is explicitly tied to `b5477211`, not a claim that all listed lines remain on today's main. Do not convert an already converted case again. The five selected cases remain source-loaded at this checkpoint; proof descriptions will follow after independent spec review. +Main at `a8e73fbf` is integrated in `74d22941`. Preserve PR #9's theme-reset conversion, the stable-front proxy's current API, SDK runtime assets, and single-origin application loading. The inventory above is explicitly tied to `b5477211`, not a claim that all listed lines remain on today's main. Do not convert an already converted case again. The five selected cases remain source-loaded at this checkpoint; the derived proofs now describe their built-application boundary and explicitly mark the adaptation pending. Main also now owns full responses from registry Vite services and selection of a contributor's origin repository for remote checkout. Preserve both. Measure built-app memory savings against this new baseline rather than attributing PR #32's benefit to PR #12. @@ -247,6 +247,6 @@ Proposal round 1 at `1652787e`: FAIL on lease scope and mixed-version consequenc Proposal round 2 at `d3570bf4`: FAIL because dead-coordinator lock recovery is undefined. The [round 2 review](test-harness-resource-cleanup-review-r2.md) is preserved unchanged. Josh selected automatic recovery, required that an orphaned active run cannot hold a worker forever, and delegated the recovery mechanism and built-app conversion scope. His supervisor clarified that the suggested predicate and grace-period wording were illustrations, not requirements. The revision at `57a701d0` supplied one renewable recovery deadline and narrowed conversion to two files. -Proposal round 3 at `57a701d0`: **PASS**, relayed by the supervisor. The [review](test-harness-resource-cleanup-review-r3.md) is preserved unchanged. Its bounded refinements are applied here: fixed recovery interval, deletion of abandoned workers despite failed process cleanup, ordinary skipping of live legacy work, and trimmed receipt/provisioning language. The [response](test-harness-resource-cleanup-review-response.md#Round 3: convergence and spec derivation) records the judgments. The proposal is converged; the next checkpoint is independent review of the derived spec deltas. +Proposal round 3 at `57a701d0`: **PASS**, relayed by the supervisor. The [review](test-harness-resource-cleanup-review-r3.md) is preserved unchanged. Its bounded refinements are applied here: fixed recovery interval, deletion of abandoned workers despite failed process cleanup, ordinary skipping of live legacy work, and trimmed receipt/provisioning language. The [response](test-harness-resource-cleanup-review-response.md#Round 3: convergence and spec derivation) records the judgments. The proposal and spec deltas have converged: spec round 1 at `cd8aa453` also passed. Its bounded refinements and the derived proof checkpoint are recorded in the response; independent proof review is next. -No proof, test, or implementation work has begun beyond integrating main. No tests, builds, remote experiments, or worker operations were performed for this checkpoint. Unknowns are live provider behavior during failed dispatch/recovery, today's worker browser versions, actual `TMPDIR` placement and post-mitigation memory savings, current capacity/credential availability, and observed recovery timing. Recovery waits at most 20 minutes after the last renewal by default, then acquisition and bounded cleanup; failed cleanup prevents test reuse and explicit pool deletion remains available. Josh's spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. +Proof derivation is complete for review; test and implementation work has not begun. No tests, builds, remote experiments, or worker operations were performed for this checkpoint. Unknowns are live provider behavior during failed dispatch/recovery, today's worker browser versions, actual `TMPDIR` placement and post-mitigation memory savings, current capacity/credential availability, and observed recovery timing. Recovery waits at most 20 minutes after the last renewal by default, then acquisition and bounded cleanup; failed cleanup prevents test reuse and explicit pool deletion remains available. Josh's spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 2f6e6d2b..fcdb8396 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -1,6 +1,6 @@ # Resource-cleanup proposal: review response -Independent round 3 reviewed `57a701d0`: **PASS**, relayed by the supervisor. The proposal has converged. The round 1 and 2 sections below retain the response history; [round 3](#Round 3: convergence and spec derivation) records the bounded refinements and current spec checkpoint. Reviewer: Claude Fable (Nina), arranged by the supervisor. +Proposal round 3 at `57a701d0` and spec round 1 at `cd8aa453` both **PASS**, relayed by the supervisor. Proposal and spec deltas have converged; the proof checkpoint below is ready for independent review. The round 1 and 2 sections below retain the response history; [round 3](#Round 3: convergence and spec derivation) records the bounded refinements and current spec checkpoint. Reviewer: Claude Fable (Nina), arranged by the supervisor. Independent round 1 reviewed `1652787e`. The [full review](test-harness-resource-cleanup-review-r1.md) remains unchanged: FAIL on two blockers. The response committed at `99b7ed93` narrowed the work pending Josh's decisions. The following sections through round 2 record that history, not the current review verdict. @@ -97,3 +97,36 @@ The rewritten spec policy on main governs this derivation. The proposal retains Validation is read-only source/policy review, exact-copy comparison of the independent review, and Git diff/whitespace inspection. No tests, builds, browser/server launches, remote experiments, or worker operations ran. Live recovery, temporary-file placement and bounded evidence transport remain to be proven on committed/pushed Blaxel revisions. The supervisor arranges independent spec review; Josh accepts the spec changes at PR time. Primitive surfaced the known recovery decisions and their later correction, alongside unrelated scope matches. Its discovery scan was truncated, so it was not a complete governing-set check. Repository policy and the supervisor's latest instructions governed the assessment; no decision was overridden or reconciled. + +## Spec round 1: convergence and proof derivation + +The supervisor relayed **PASS** on `cd8aa453`. The [complete spec review](test-harness-resource-cleanup-spec-review-r1.md) is copied verbatim. Apply the nonblocking refinements under the bounded rule; no additional spec-review round is requested by this response. + +| Refinement | Assessment and disposition | +| --- | --- | +| 1: terminology | Use lease for coordinator ownership and lock for its directory. Say shard attempt for one shard's execution, retaining attempt for the shard-set attempt. Keep the existing block anchor stable. | +| 2: unreadable records | Use the same clean-audit requirement for acquisition and deletion. Missing ownership evidence authorizes neither signalling by name nor ignoring surviving owners. | +| 3: command-unavailable deletion | Partly accepted. A fresh worker never published for dispatch can be deleted without a lease. Do not adopt the broader inference that a worker absent from coordinator discovery has no live or valid earlier run: the review supplies no provider guarantee for that implication. A previously eligible worker whose ownership check cannot run remains held/reported for operator repair. This preserves the converged no-live-lease deletion rule; it adds no new recovery service or override. | +| 4: helper rule | Trim the spec to worker ownership, publication after success, teardown and the Blaxel backstop. The proof derives pending-build/failure cleanup and real Playwright/Vitest crossings from that rule and ordinary cleanup correctness. | +| 5: operations and output | State idle-pool deletion/refusal consequences, locate resource sidecars and receipts under downloaded artifacts, and name the temporary-cleanup lifecycle step. | +| 6: transitional text | Label the legacy protocol transitional and remove the two non-requirements. | +| 7: testing guidance | Keep actual Node and Chromium descriptor observations. Remove the duplicate placement rule and leave browser-version measurement hygiene in the proposal's experiment plan. | +| 8: sidecar lifetime and wording | Explicitly finalize setup sidecars outside scratch before removal; keep allocation-versus-free-space interpretation in working guidance rather than authority. | + +### Proof checkpoint + +The mirrored proofs now derive the following coverage, with new obligations explicitly marked **test to be written**: + +| Owner | Coverage and cost boundary | +| --- | --- | +| `proofs/arch/test-runner/blaxel-testshards.md` | Contract decisions/order; real guard/registration and filesystem seams; canonical live retrieval/refusal, automatic active/quiet recovery, fresh provisioning and pool deletion on dedicated workers. Planned and targeted paths share small fixtures. Node and actual Chromium descriptor paths establish temporary-directory inheritance. No deliberate OOM, disk exhaustion, shared-pool contamination, or matrix of provider failures. | +| `proofs/arch/test-runner/reporting.md` | Bounded writer and retained-link contracts; real parent/Linux/native-runner capture; downloaded failed and recovered-attempt evidence composed into the same provider runs. Unknown probes, missing observations and lost reports cannot become passing evidence. | +| `proofs/arch/testing-policy.md` | Worker cache/publication contract, real Playwright and Vitest teardown seams with real source-CLI/web output, and failed-launch copy cleanup. The built-app default is honored by test design, not a policy-compliance source scan. | +| Shared-execution and runner proofs | Cross-reference the owner-token, temporary-directory, diagnostic-history and real-plan crossings above. Keep ordinary stale-owner protection and isolated-GitHub fixture restrictions. | +| Theme product/delivery and artifact product proofs | Update the selected five cases' loading boundary and mark adaptation pending. Preserve their user-visible assertions, stable origin/authentication over restart, and the separate Settings development-module walk. PR #9's reset remains unchanged. | + +The main risk worth the live cost is reuse before owned processes stop or evidence reaches the caller. Contracts cover failure permutations, but cannot replace real Blaxel metadata updates, dispatch, renewal, cleanup and download. The two private workers and separate Blaxel coordinator in the proposal remain planned resources, not an assertion that they exist or are available. Actual execution will use committed/pushed revisions and the canonical runner; native retries retain the default budget. Only the existing recovery-interval option is shortened deliberately for crash tests. + +Validation for this checkpoint is document/source inspection, review-copy comparison, Markdown index regeneration, and Git whitespace/diff inspection. No tests, builds, browser/server launches, provider experiments, or worker operations ran. Test and harness implementation has not begun. Independent proof review is the next gate; Josh's spec review remains at PR time. + +Primitive's file check surfaced existing recovery decisions and the correction to the supervisor's illustrative rule, with `truncated: true` from discovery. That was not a complete constraint check; no decision was overridden or reconciled. These proof details derive the existing choices and introduce no new higher-order direction to record. diff --git a/docs/working/test-harness-resource-cleanup-spec-review-r1.md b/docs/working/test-harness-resource-cleanup-spec-review-r1.md new file mode 100644 index 00000000..1f628fd8 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-spec-review-r1.md @@ -0,0 +1,65 @@ +# Spec review, round 1: test-harness resource cleanup (PR #12, cd8aa453) + +**Verdict: PASS.** Every decision in the converged proposal is in a spec, in the owner the proposal named, and the affected tree reads consistently: the Blaxel spec's lifecycle, the reporting contract, the shared execution model, the runner's owner rules and the testing policy agree with each other and with the unchanged specs around them. Nothing in another spec still promises the mechanism this change replaces. Eight refinements follow; the first two are worth making before Josh reads the deltas, and the rest are trims that keep the specs as simple as the proposal intends. + +- Reviewed: `git diff origin/main...cd8aa453 -- specs` (five specs and the regenerated index), against the proposal and response at `cd8aa453`, the reviewer checklist, main's rewritten spec policy, the unchanged specs and proofs that cite the replaced mechanism, and the code the specs describe. +- Branch state: the branch contains `origin/main` (`a8e73fbf`); the merge `74d22941` changes nothing outside `docs/working/` relative to main and nothing in `docs/working/` relative to the branch. The copied round 3 review is identical to mine. +- I ran nothing. Every check is a read of files in the checkout or of git history. + +## The proposal's decisions, located + +| Decision | Spec | Checked | +| --- | --- | --- | +| One lease through teardown and download; shell never unlocks | `blaxel-testshards.md:45-47` `^blaxel-release` | Matches proposal order 1-5. | +| Coordinator-only renewal; 20-minute interval via `--lock-ttl-seconds`, independent of the execution timeout | `:41`, options table `:126`, profiles `:135` | 1200 s and "20 minutes" agree; the old TTL rows are gone. | +| Automatic recovery: one sequence for quiet and active leftovers, kill by token, discard orphan reports, receipt | `:51-57` `^blaxel-recovery`; receipt contract `reporting.md:224` | Receipt shares the 8 MiB attempt budget; no separate cap. | +| Attempt owner token preserved beside the surface token; registration before checkout | `:43`; `test-runner.md:222` | Consistent with the one-variable token the supervisor replaces today (`scripts/test/surface-supervisor.mjs:62`). | +| Legacy records: `expiresAt: 0`, respect TTL, skip live legacy work | `:61` | Matches the old reclaim test (`run-blaxel-testshards.mjs:406`) and the matching-ID trap (`:773-775`). | +| Deletion may destroy an expired worker that failed cleanup; dispatch eligibility removed first | `:63` | The SDK can change labels on a live sandbox (`@blaxel/core` `SandboxInstance.updateMetadata`). | +| Labels published after provisioning; no silent reuse; `ensure` does not refill | `:33` | Same SDK call. | +| `--audit-stale-owners` reserves without recovery | `:65`, table `:133` | | +| Per-attempt scratch, `TMPDIR`, exact two-prefix cleanup, every non-current attempt directory | `:69-80` `^blaxel-scratch`, `^blaxel-known-leftovers` | Caches, lock, guard, uploads and X11 preserved. | +| Minimum diagnostics, retention rules, 64 KiB / 128 KiB / 8 MiB bounds, honesty about what the numbers mean | `reporting.md:214-226` `^resource-diagnostics`, `^resource-diagnostic-bounds`; fields in `reporting.md:85` and `sharded-execution.md:255` | | +| Worker-owned reusable builds | `testing-policy.md:210` `^worker-build-lifetime` | See refinement 4. | +| Built application by default for `ProductServer` acceptances, scoped | `testing-policy.md:212` `^product-acceptance-built-app` | Excludes `appURLForServer()`, fixtures and Electron seams, as converged. | + +Other checks: the nine new anchors are unique in the tree; every new link resolves to an existing heading (`Resource diagnostics`, `Abandoned-run recovery`, `Locking`, `Temporary files`, `Destructive fixture placement`, `Pool`); the index's line counts match the files; `preflight.md:47`, `test-runner.md:153` and `sharded-execution.md:321` still describe the stale-owner exception correctly; the test-runner proof's "retains the lease until TTL" (`proofs/arch/test-runner/blaxel-testshards.md:57`) is the expected proof-stage follow-up, which the response defers on purpose. + +## Refinements (none blocking) + +1. **One word for the thing the coordinator holds.** The Blaxel spec now says *reservation* 17 times, *lease* 14 and *lock* 14 for the same object, and `test-runner.md:153` and `sharded-execution.md` still say *lease*. A cold reader will look for a distinction that does not exist. Keep *lease* (the established term) for what the coordinator holds and *lock* for the directory that records it, and drop *reservation*. The same applies to *attempt*: `terms.md:124` defines it as one execution of the shard set, while the new text uses it for one shard's execution (`attempt directory`, `attempt owner token`, `incoming attempt`). Say *shard attempt* where that is meant, as `reporting.md:218` already does. + +2. **An unreadable lock record is treated two ways.** `blaxel-testshards.md:57` lets ordinary acquisition reclaim an acquisition interrupted before registration after a clean worker audit, since execution cannot precede registration. `:63` says that for deletion "an unreadable ownership record is not proof of expiry", so the pool script would refuse a worker a test run may take. One rule serves both: a record that cannot be read is reclaimed, by acquisition or deletion, only after a clean worker audit, and held otherwise. Say it once, in the recovery section, and let the deletion paragraph refer to it. + +3. **Deletion of a worker that cannot run commands is undefined.** Deletion "checks the same identity and deadline under the guard", which needs a worker that answers `process.exec`. A sandbox that is not `DEPLOYED` cannot be leased (`:25`), so nothing can be running on it and it can be deleted without a guard; a `DEPLOYED` worker whose commands fail is the provider failure the spec sends to an operator, and deletion is that operator's tool. One sentence closes this: a worker no coordinator can lease may be deleted without a reservation. + +4. **`^worker-build-lifetime` says more than the rule.** The need is one sentence: a reusable build a helper creates belongs to the worker that created it, is published only after it succeeds, and is removed at worker teardown rather than per test case. The rest, waiting for pending builds, stopping consumers, Playwright and Vitest both, the per-launch copy and home, idempotence, is what a competent implementer does without being told, and each clause is a line every future helper change is reviewed against. Trim to the rule and the backstop link. + +5. **Consequences for the unchanged operations text.** `:158` still says to provision `poc` by deleting the complete pool, which under `:63` now refuses any worker with a live lease. Say that recreation needs an idle pool, or that refused workers are reported and the command exits non-zero. Relatedly, the `Output` section (`:141`) lists what a shard download adds but not where resource sidecars and recovery receipts land; one clause (under `artifacts/shard-/`, with the task artifacts, as `sharded-execution.md:228` implies) saves an implementer a guess, and the shard lifecycle's step list (`:86`) should name the cleanup step if its counts and free-space figures are recorded as a setup timing, as `:76` requires. + +6. **Mark the legacy paragraph transitional and drop its non-requirements.** `:61` is a mixed-version rule that ends when every branch has integrated this change. Saying so lets a later change delete it with confidence. Its last sentence, "No pool-wide drain or simultaneous branch update is required", and `:63`'s "Read-only pool listing needs no reservation" state what the spec does not require, which spec policy leaves to silence. + +7. **Two items in the Blaxel `Testing` section fail the testing-guidance test.** "Existing `isolated-github-only` fixtures retain their placement restriction" restates a rule that applies regardless (`test-runner.md:191`); removing it changes no derivation. Recording the launched browser version and executable is investigation hygiene for interpreting measurements, not something a proof needs to prove `^blaxel-scratch`; keep the `TMPDIR` observation, which a test author would miss, and move the browser-version note to the proposal's validation plan where it already lives. + +8. **Small wording.** `:69` says setup sidecars "use the attempt directory" and `:78` removes that directory before download; the two agree only through "finalize evidence outside scratch" between them. Say that setup sidecars are written in the attempt directory and finalized into the results tree before it is removed. `:76`'s last sentence ("Do not equate allocated bytes removed with free-space change…") is a caution about reading the numbers, not a requirement; the proposal can keep it. + +## Reviewer checklist + +- **Slop-free / authority.** The deltas stay in the owners the proposal named; the Blaxel spec's opening line and paragraph now claim recovery and temporary files, and reporting's opening claims resource diagnostics. The TTL escape and shell-exit unlock are replaced, not left beside the new rule. +- **Altitude and contract vs. implementation.** The TypeScript fields are the only type change and match on both sides. The Blaxel spec is where the lifecycle detail belongs; the testing policy is one place where the detail outruns the rule (refinement 4). +- **Writing and terminology.** Refinement 1. Otherwise plain, with rationale kept short. +- **Testing guidance.** Refinement 7. The new testing-policy default is a default, not a prohibition, and is honored by test design rather than asserted by tests. +- **References.** All new links and anchors resolve; nothing links to the removed sentences. +- **Size.** The Blaxel spec grows from 131 to 174 lines, in proportion to what it now decides; it is not a decomposition candidate. +- **Status.** The specs promise behavior the coordinator does not yet have, which is the workflow's normal state between spec and implementation. The one transitional rule is in refinement 6. +- **Proof coverage and mocking.** Not at this stage; the response is right that proofs follow spec convergence. + +## Evidence and limits + +- Read the full spec diff and the full Blaxel spec at `cd8aa453`; the proposal and response diffs since `57a701d0`; the merge commit and its content relative to both parents; spec policy's testing-guidance and transitional-status rules; `terms.md`; the preflight, test-runner, sharded-execution and reporting passages that cite leases or stale owners; the test-runner proof line that cites TTL; and the SDK's sandbox update methods. +- Not established: whether Blaxel accepts a label change on a `DEPLOYED` sandbox in practice (the SDK exposes the call); whether recovery fits inside the coordinator's current 30-second acquisition command timeout (an implementation choice the spec does not constrain). + +## Housekeeping + +- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel. `git fetch origin` updated remote-tracking refs. +- Primitive produced no output during this review, so I have nothing to report on its usefulness. The response records Primitive output in the author's session; I have not seen it and do not assess it. diff --git a/proofs/arch/test-runner/blaxel-testshards.md b/proofs/arch/test-runner/blaxel-testshards.md index 58b93ef0..f1e0531a 100644 --- a/proofs/arch/test-runner/blaxel-testshards.md +++ b/proofs/arch/test-runner/blaxel-testshards.md @@ -8,7 +8,11 @@ Proves [specs/arch/test-runner/blaxel-testshards.md](../../../specs/arch/test-ru The host marker’s provider-selection behavior is covered by [the runner proof](test-runner.md#^t-blaxel-host-marker). The corporate credential location is setup information checked by human review, not a new token-source mechanism. The existing token-source assertions remain unchanged; no test reads the vault or asserts its item name. -Coverage declarations are carried inside the migrated assertion blocks below. +Lease decisions and failure ordering are checked cheaply at the worker/coordinator contracts. Real Linux processes and files cross the guard, owner-registration, and deletion seams. These tests do not claim Blaxel transport coverage. + +The live acceptances below enter through the canonical CLI on a dedicated pool: normal retrieval, refusal while another coordinator holds the worker, recovery after coordinator death, and pool creation/deletion. They share a small committed fixture selection and fresh workers rather than repeating a full suite for every failure permutation. Both planned and targeted execution paths are crossed. Resource retention composes with [reporting's diagnostics assertions](reporting.md#Resource diagnostics); helper teardown belongs to [the testing-policy proof](../testing-policy.md#Worker-owned builds). + +The spec's [Testing](../../../specs/arch/test-runner/blaxel-testshards.md#Testing) directives require real API/transport, fresh provisioning, and observed Node/Chromium temporary paths. Private workers and bounded disposable processes supply those observations; no deliberately exhausted host or shared-pool contamination is needed. The live tests are an explicit paid-service selection, not a recursive addition to every broad run. Mocks below forfeit those crossings to the named live assertions. New assertions are obligations, not evidence of runs already performed. `test/repo/blaxel-stale-owner-audit.test.ts` checks diagnostic construction and coordinator ordering on every host. Its live diagnostic cases require Linux procfs and run only on Linux; they establish no macOS backend coverage. `test/repo/test-runner-provider.test.ts` exercises shell activation ordering with an authored nvm replacement in a shell isolated from host startup files. That contract check does not install nvm or establish real worker activation. @@ -27,11 +31,17 @@ Pool discovery also has a contract test that runs the shared discovery helper us The SDK reads `BL_API_URL`, `BL_API_KEY`, and `BL_WORKSPACE`; the discovery contract test supplies a loopback API and fixture credentials in an isolated working directory and home, with an explicit child environment that preserves the test owner token and disables SDK telemetry and HTTP/2. Production uses the configured Blaxel API and credentials. +The worker lock and temporary-cleanup helpers accept filesystem roots for disposable seam fixtures; production uses the specified worker paths. Their decision layer accepts a clock, owner-audit/signal operations, and filesystem probes for contract failure injection; production uses worker time, procfs, signals, and the filesystem. + +A coordinator milestone hook, enabled only by `TV_TEST_RUNNER_SELFTEST`, can wait on a fixture-controlled rendezvous immediately before report download while renewal continues; ordinary runs do not wait. It changes scheduling, not lease state, process results, or report bytes. Live recovery uses the existing `--lock-ttl-seconds` override to shorten the abandonment wait, not a fake clock or altered retry budget. + +Pool-management contract tests may supply recording/failing SDK operations; ordinary pool commands use the installed SDK. These substitutions forfeit provider behavior to the live pool acceptance below. + ## Assertions ### Test assertions -- Pool discovery includes matching workers on later API pages and excludes workers with other project, purpose, pool, or architecture labels, as required by the spec's [Pool](../../../specs/arch/test-runner/blaxel-testshards.md#Pool) contract. **Contract:** the consumer is the shared discovery helper and installed SDK, called in a child process; the service is the HTTP mock described above. The helper collects the unfiltered listing before filtering locally. Both the manager mode (all matching workers) and coordinator mode (only matching `DEPLOYED` workers) must traverse all pages, including an earlier page containing no matching workers. A page-fetch failure rejects discovery rather than returning a partial pool. Fixture sandbox records omit a region so discovery needs no data-plane connection. (policy-grade test: `test/repo/blaxel-pool-discovery.test.ts`) ^blaxel-pool-pagination +- Pool discovery includes matching workers on later API pages and excludes workers with other project, purpose, pool, or architecture labels, as required by the spec's [Pool](../../../specs/arch/test-runner/blaxel-testshards.md#Pool) contract. **Contract:** the consumer is the shared discovery helper and installed SDK, called in a child process; the service is the HTTP mock described above. The helper collects the unfiltered listing before filtering locally. Both manager and coordinator modes must traverse all pages, including an earlier page containing no matching workers. The coordinator sees only published, matching `DEPLOYED` workers; the manager also retains the pool's unpublished and partially provisioned workers. A page-fetch failure rejects discovery rather than returning a partial pool. Fixture sandbox records omit a region so discovery needs no data-plane connection. (policy-grade test for existing pagination: `test/repo/blaxel-pool-discovery.test.ts`; publication/partial-provisioning extension: test to be written) ^blaxel-pool-pagination The repository-selection assertions: @@ -54,7 +64,33 @@ The remaining coverage lives in `test/repo/test-runner-provider.test.ts`. - Provider summaries and normalized run events preserve each shard's restore/wake, checkout, dependency cache outcome, readiness, surface compute, and report-download phases without parsing human logs. - The normal-lease acquisition scheduler never exceeds its concurrency bound, attempts each requested shard exactly once, and preserves input order in its outcomes. A partial batch writes a provider report containing acquired-but-not-run, unavailable, and thrown-error shard outcomes before releasing every partial success. - Signal shutdown writes completed, running, and pending shard outcomes before lease cleanup; a report-write failure does not suppress cleanup. -- Unconfirmed ordinary execution cleanup orders named stop, owner-token reaping, confirmed zero-owner state, then lock release; failed owner confirmation retains the lease until TTL. Completed and undispatched paths do not run descendant cleanup. +- Ordinary cleanup/release coverage is superseded by [the lease ordering contract](#^blaxel-t-lease-order) and [the live retrieval acceptance](#^blaxel-t-retrieval); shell completion alone no longer permits release. - A stale-owner setup failure records the contaminated sandbox and its lifecycle evidence, excludes that sandbox from subsequent attempts, and permits a configured infrastructure retry only on a distinct sandbox that independently passes lease acquisition and owner audit. -**Boundary gaps — stated honestly per [testing-policy.md](../../../specs/arch/testing-policy.md):** the classifier, normalization, bounded acquisition scheduler, and provisioning/activation/cache command shapes are covered without calling Blaxel. The scheduler test uses an injected acquisition callback and therefore does not cross the Blaxel API boundary. The code-only checks cannot prove nvm installation, selected Node/npm versions, or cache behavior inside the base image. After review, deleting and recreating `poc` and running an operator-approved targeted shard provide that seam evidence along with the real coordinator path: leasing sandboxes, lock protocol, architecture validation, checkout/runtime/dependency setup, streaming logs, and result download. There is no recorded-fixture stand-in for that external boundary today. Repository switching on real workers has no committed test; the live runs from two repositories' checkouts described in the coverage model provide that evidence for a repository-selection change. +**Boundary gaps — stated honestly per [testing-policy.md](../../../specs/arch/testing-policy.md):** the classifier, normalization, bounded acquisition scheduler, and provisioning/activation/cache command shapes are covered without calling Blaxel. The scheduler test uses an injected acquisition callback and therefore does not cross the Blaxel API boundary. Code-only checks cannot prove installation or operation inside the base image. The live assertions below must supply that evidence; their pending citations must not be read as a passed provider run. Repository switching on real workers has no committed test; the live runs from two repositories' checkouts described in the coverage model remain required for a repository-selection change. + +### Lease, recovery, and scratch + +- **Contract — lease decisions.** The shared lock decision layer consumes authored current, expired, legacy, incomplete, and unreadable records with a supplied worker clock and audit/signal results. Under [Locking](../../../specs/arch/test-runner/blaxel-testshards.md#Locking) and [recovery](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery), renewal preserves ownership only for the matching unexpired lease; the default recovery interval is 1200 seconds regardless of execution timeout. Expiry permits cleanup, not dispatch: both quiet and active abandoned runs reach a new lease only after confirmed cleanup and a clean audit. Unknown identity or surviving owners hold the worker. Incomplete/unreadable records have the same clean-audit requirement for acquisition and deletion. Legacy positive TTLs are respected, live legacy owners are skipped, and new records use `expiresAt: 0`. Audit mode never recovers or deletes residue. Mock clock and audit/signal operations forfeit real serialization and termination to the guard seam and live recovery below. (test to be written) ^blaxel-t-lease-decisions + +- **Contract — coordinator ordering and failures.** The coordinator's lease lifecycle uses recording/failing transport, download, and cleanup callbacks. Under [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release), renewal spans acquisition, execution, cleanup, and retrieval. A successful shell exit still requires descendant confirmation; unknown dispatch or failed cleanup cannot release. Successful retrieval and identity validation precede release; a download retry neither reruns tests nor resets worker results. An unused lease waits for pending writes and retains produced evidence. On interruption, the authoritative interrupted report precedes cleanup/retrieval and cannot become an ordinary completion report; cleanup still runs after a report-write error. A coordinator that loses ownership stops issuing mutations and reports infrastructure failure. These callbacks forfeit API, signals, and downloads to the live retrieval/recovery assertions; existing signal-report fixtures supply breadth without another native suite. (test to be written) ^blaxel-t-lease-order + +- **Seam — guarded registration and ownership.** Production worker-entry and lock helpers cross real Linux process/filesystem boundaries in a private fixture root on a dedicated worker. Two competing acquisitions yield only one owner; killing a process holding the short guard releases the OS guard without deleting another lease. A request paused before entry registration, then resumed after ownership changes, cannot touch a checkout sentinel. Old-ID renewal/release cannot change the new record. Registered setup descendants preserve the outer token when a nested surface replaces its own token; recovery stops only the attributed PID/start identities, including a descendant that ignores TERM and needs KILL. A separate live fixture owner remains untouched and prevents reuse. The fixtures are small processes, records produced by the real helper, and rendezvous files. The supplied audit filters real procfs observations to the fixture tokens, a mock of whole-worker scope that forfeits that protection to live recovery; process inspection, identity revalidation, signals and filesystem operations remain real. This proves [registration](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-reservation) and [recovery](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery), not provider dispatch. (test to be written) ^blaxel-t-worker-guard + +- **Seam — bounded filesystem cleanup.** The production cleaner crosses the real filesystem in a disposable parent containing both allowed legacy prefixes, stale scratch with complete and missing metadata, the current scratch, unrelated files/directories, and symlinks to outside sentinels. With a lease and clean-audit precondition supplied as a mock, it removes exactly the permitted real directories, leaves the current directory and outside sentinels intact, rejects symlink roots, and records counts, allocated bytes and free-space observations under [known leftovers](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-known-leftovers). The audit mock forfeits real disuse confirmation to the live retrieval/recovery cases. No assertion equates allocated bytes with the filesystem's free-space increase. (test to be written) ^blaxel-t-scratch-cleanup + +- **Contract — cleanup uncertainty.** The cleaner's decision layer consumes authored filesystem-probe results and removal errors. An unexpected mount boundary is skipped, and an incomplete removal is reported without claiming the whole directory was reclaimed; neither permits new test dispatch under [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#Temporary files). These mocked conditions forfeit actual filesystem operations to the cleanup seam and live retrieval assertion; a real mount or full disk adds no necessary coverage. (test to be written) ^blaxel-t-scratch-errors + +Shared live declaration: the following acceptances spawn the canonical CLI with real argv and inspect its exit and retained run directory. A fresh dedicated pool, committed native fixture tests, known leftover directories and unrelated sentinels are fixtures. The coordinator, SDK, worker scripts, native runners, transport, ownership audits and filesystem operations are real. The report-download rendezvous above is the only scheduling hook; no mechanism is mocked. A bounded test driver owns and finally cleans up its coordinators and private workers. These tests do not invoke fixtures restricted to isolated GitHub hosts. + +- **Acceptance — reports before reuse.** A small planned run with at least two shards traverses real acquisition, architecture verification, checkout, runtime/dependency setup, execution and log/report transport under [the shard lifecycle](../../../specs/arch/test-runner/blaxel-testshards.md#Shard lifecycle). While it waits at the download rendezvous, observed renewals keep the leases valid; a competing canonical run cannot acquire the occupied capacity or alter its result files. After retrieval, the original native reports, setup/cleanup evidence and diagnostics have the original identities and readable local paths, scratch is gone, and the workers become available to a targeted run. Legacy-prefix and stale-scratch fixtures disappear, unrelated sentinels and persistent caches survive, and the next run does not accumulate an older results tree. The targeted run crosses its distinct dispatch path too. This proves [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release) and [scratch lifetime](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-scratch); the shared diagnostics observations are owned by [reporting](reporting.md#^reporting-t-resource-live). (test to be written) ^blaxel-t-retrieval + +- **Acceptance — abandoned run recovery.** In two small scenarios, kill the real coordinator: once while a shard supervisor and a token-bearing child remain, and once after the shard stops but before download. The next canonical run, after the shortened configured recovery interval, stops the old owned processes where present, confirms a clean worker, clears allowed residue, and runs on that worker. Its downloaded recovery receipt retains the old identity, renewal/deadline, cleanup result and discarded-report flag across results reset. No old report becomes a pass. This proves the same [automatic recovery path](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery) works for active and quiet orphans without waiting for their voluntary exit. Failure permutations are carried by the contracts, not more live crash runs. (test to be written) ^blaxel-t-recovery-live + +- **Seam — temporary-directory inheritance.** In the live planned and targeted fixture runs above, the shard environment reaches real Node and Chromium: Node's `os.tmpdir()` and created files are inside the current `tmp/`, and actual Chromium shared-memory descriptor paths resolve there, including unlinked files. Inspect the launched browser's descriptors on the worker; an environment-variable echo alone is insufficient. Nested surface tokens must not erase the outer ownership token or `TMPDIR`. After process teardown, current scratch is absent while retained sidecars are readable. This honors [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-scratch) and [Testing](../../../specs/arch/test-runner/blaxel-testshards.md#Testing). A small HTML page is the browser fixture; no browser or path mechanism is mocked, and no memory-pressure or exact byte-saving claim is made. (test to be written) ^blaxel-t-tmpdir-live + +### Pool mutation + +- **Contract — publication and deletion ordering.** The pool manager uses authored sandbox states and recording/failing SDK operations. Under [Pool](../../../specs/arch/test-runner/blaxel-testshards.md#Pool) and [pool operations](../../../specs/arch/test-runner/blaxel-testshards.md#Older runners and pool operations), it publishes eligibility only after successful provisioning/verification, refuses to refill partial pools, refuses deletion of valid leases, and unpublishes before deleting an idle or expired worker. An expired lease can be deleted despite failed process cleanup; a failed deletion stays unavailable and visible. Unreadable records follow the common contract above; command failure on a previously eligible worker holds/reports it, while a fresh worker never published may be deleted without a lease. SDK substitutions forfeit actual metadata changes and deletion to the live pool acceptance. (test to be written) ^blaxel-t-pool-order + +- **Acceptance — fresh pool and safe deletion.** Through canonical pool commands and the real API, create the dedicated pool used by the live assertions, observe workers become eligible only after runtime/architecture verification, and run the fixtures on them. Attempt deletion while the retrieval gate holds valid leases: the command reports refusals and exits nonzero, and the original run still completes. After release, delete the pool and confirm its workers are absent; a new coordinator cannot dispatch there. This proves the [pool mutation](../../../specs/arch/test-runner/blaxel-testshards.md#Older runners and pool operations) and [operations](../../../specs/arch/test-runner/blaxel-testshards.md#Pool provisioning) spine through actual Blaxel label changes and deletion. These observations share the retrieval scenario rather than provisioning another pool. (test to be written) ^blaxel-t-pool-live diff --git a/proofs/arch/test-runner/reporting.md b/proofs/arch/test-runner/reporting.md index a214d39b..eb3d99b4 100644 --- a/proofs/arch/test-runner/reporting.md +++ b/proofs/arch/test-runner/reporting.md @@ -6,7 +6,11 @@ Proves [specs/arch/test-runner/reporting.md](../../../specs/arch/test-runner/rep ## Coverage model -Coverage declarations are carried inside the migrated assertion blocks below. +Native-report contracts carry normalization breadth; real native-runner and CLI crossings carry attempt capture and final output. The existing declarations below name those boundaries. + +Resource diagnostics compose a bounded writer contract, a real Linux probe/parent-lifecycle seam, and the dedicated Blaxel acceptances owned by [the provider proof](blaxel-testshards.md#Lease, recovery, and scratch). A small native failure and a recovered retry establish retained evidence; they do not reproduce resource exhaustion. Authored unavailable/reset/oversized probe inputs cover uncommon diagnostic failures without filling a disk or causing an OOM kill. Those mocks forfeit kernel reading and transport to the real seams. + +Planned and targeted execution both cross parent capture and download. The same sidecars pass through provider normalization and attempt history; a recovered infrastructure attempt must not replace earlier evidence. Recovery receipts are observed in the provider's crash acceptance, not a duplicate crash here. No test claims start/end samples measure a peak or that an unavailable kernel log excludes OOM. The new resource assertions remain pending until implemented and run. ## Test hooks @@ -15,6 +19,8 @@ Coverage declarations are carried inside the migrated assertion blocks below. `TV_VITEST_ATTEMPT_FILE` supplies the NDJSON output path to the production Vitest attempt recorder. The canonical runner and the planned shard worker set it to the current native run's attempt-result path. A Vitest invocation that configures the recorder outside those entrypoints must set it; the recorder fails without it. +The resource collector accepts a probe reader and clock; contract tests provide authored readings/errors and times, while production reads Linux counters, memory information and filesystem statistics at the current time. The writer accepts an output root and write operation; tests use disposable directories and may inject a write failure, while production writes into the current results tree. Native fixture selection uses the runner's existing guarded registry hook, not a replacement reporter or fabricated native result. + ## Assertions ### Test assertions @@ -40,3 +46,12 @@ The event-normalization assertions are **contract tests** over recorded native-r - A planned shard worker running under GitHub Actions finalizes its own run directory and retains its shard summary, plan, native results, and logs inside it. - A repository workflow assertion pins full run-directory artifact upload on success and failure and a 30-day artifact retention period. +### Resource diagnostics + +- **Contract — bounded, honest observations.** The collector/writer consumes authored start/end memory, filesystem, OOM-counter and kernel-line readings, including unavailable probes, a counter reset and over-budget excerpts, and writes real files in a temporary results root. Under [resource diagnostics](../../../specs/arch/test-runner/reporting.md#^resource-diagnostics) and [bounds](../../../specs/arch/test-runner/reporting.md#^resource-diagnostic-bounds), values keep units/time, unknowns keep reasons rather than zero, and resets produce no negative OOM delta. A failed native attempt, even if later recovered, or incomplete execution requests the interval's kernel evidence. The bounded reader/writer retains at most 64 KiB of kernel lines and 128 KiB per sidecar, and 8 MiB total per shard attempt including receipts; compact start/end evidence survives before optional excerpts, with explicit truncation. Only allowed fields are serialized: supplied credential, query/body, environment and argument sentinels do not appear. Injected probe and write failures warn without changing the original native outcome. Mock readings/write failures forfeit OS reading to the parent seam and actual download to the live assertion below. (test to be written) ^reporting-t-resource-bounds + +- **Seam — parent capture survives the child.** The production surface parent crosses real Linux procfs/filesystem probes and an installed native runner's child-process boundary in a disposable results root. Authored tiny native fixture tests produce a failure/retry and an exit before native output; both planned-worker and targeted execution wiring are exercised. The retained baseline exists before preparation begins and the final observation follows teardown; a terminated child cannot erase the baseline. A separate interrupted-parent fixture leaves the baseline with an explicitly missing final observation rather than invented zeroes. Real probe values are checked for their source/units and availability reasons, not exact memory totals. No kernel OOM is manufactured; inaccessible kernel evidence must remain unknown. No mechanisms are mocked. This proves the capture boundary in [resource diagnostics](../../../specs/arch/test-runner/reporting.md#^resource-diagnostics); the live assertion carries remote retrieval. (test to be written) ^reporting-t-resource-parent + +- **Contract — retained links and attempt history.** Provider normalization and infrastructure-attempt aggregation consume authored shard/task envelopes referring to real temporary sidecar files, with recorded native failed/recovered fixtures. Under [normalized results](../../../specs/arch/test-runner/reporting.md#Normalized surface result) and [resource diagnostics](../../../specs/arch/test-runner/reporting.md#Resource diagnostics), each advertised `resourceDiagnosticsPaths` entry exists relative to the final run directory; earlier attempt evidence survives a later successful infrastructure attempt. Missing native output remains incomplete and cannot manufacture a selected-surface result. Samples and kernel excerpts appear in neither compact console output nor timing events. Mocked provider envelopes forfeit remote creation/download to the live assertion; recorded native inputs forfeit attempt capture to the real parent seam and existing native-recorder assertions above. (test to be written) ^reporting-t-resource-links + +- **Acceptance — evidence reaches the caller.** Share the dedicated-pool canonical CLI runs in [the provider retrieval acceptance](blaxel-testshards.md#^blaxel-t-retrieval), with a committed fail-once native fixture that recovers under the default retry budget and a fixture that fails all attempts. Planned and targeted runs retain the native outcomes and their start/end sidecars after worker scratch is removed; every normalized diagnostic link opens a downloaded file with the correct run/shard/surface identity and native-attempt references. The failed run still fails, the recovered case retains both attempts, and any inaccessible kernel evidence says so. The provider's [crash acceptance](blaxel-testshards.md#^blaxel-t-recovery-live) supplies the downloaded recovery receipt and its disclosed report loss. Boundary: real CLI, Blaxel API, native runners and report files, with the provider proof's scheduling hook and no mocks. This proves [resource retention](../../../specs/arch/test-runner/reporting.md#Resource diagnostics) end to end without a resource-exhaustion experiment. (test to be written) ^reporting-t-resource-live diff --git a/proofs/arch/test-runner/sharded-execution.md b/proofs/arch/test-runner/sharded-execution.md index 8f289374..a69970fc 100644 --- a/proofs/arch/test-runner/sharded-execution.md +++ b/proofs/arch/test-runner/sharded-execution.md @@ -8,6 +8,8 @@ Proves [specs/arch/test-runner/sharded-execution.md](../../../specs/arch/test-ru Coverage declarations are carried inside the migrated assertion blocks below. +The [Blaxel guard seam](blaxel-testshards.md#^blaxel-t-worker-guard) and [live temporary-directory seam](blaxel-testshards.md#^blaxel-t-tmpdir-live) prove that the outer shard attempt's ownership and `TMPDIR` survive the shared worker and nested surface environments required by [Shared shard worker](../../../specs/arch/test-runner/sharded-execution.md#Shared shard worker). The [planned live acceptance](blaxel-testshards.md#^blaxel-t-retrieval) also supplies real plan transfer. Diagnostic artifact fields and attempt-history retention in [the provider model](../../../specs/arch/test-runner/sharded-execution.md#Blaxel provider report shape) compose with [reporting's link contract](reporting.md#^reporting-t-resource-links) and [live retrieval](reporting.md#^reporting-t-resource-live). Those assertions remain pending; this proof orders no duplicate provider runs. + ## Test hooks @@ -49,4 +51,3 @@ Planner and baseline assertions are **contract tests** over temporary registries - Shard-count recommendations select the smallest simulated count under 120 seconds, obey the supplied capacity, and flag an atomic file over target. The baseline update reports only the Blaxel recommendation, simulated up to the Blaxel pool size. **Boundary statement:** real local runner processes prove explicit file selection for Vitest and the default Playwright command, report shape, and join logic. They do not prove that the Electron wrapper consumes a generated Playwright list, that a Blaxel worker accepts the transferred plan, or that independently started GitHub matrix jobs derive the same plan on hosted machines. Live Electron, Blaxel, and GitHub runs must validate those execution boundaries before activation. - diff --git a/proofs/arch/test-runner/test-runner.md b/proofs/arch/test-runner/test-runner.md index 44eef1fc..cfd256d1 100644 --- a/proofs/arch/test-runner/test-runner.md +++ b/proofs/arch/test-runner/test-runner.md @@ -10,6 +10,8 @@ Coverage declarations are carried inside the migrated assertion blocks below. Pr The [combined-project seam](sharded-execution.md#^combined-vitest-projects) crosses the shared `vitestProjectsArgs` handoff that writes the combined config and selects it for the Vitest command used by local and sharded runs. It exercises project roots, exact file selection and attempt recording. The local wrapper's pre-commands, execution without an explicit file list and result splitting remain within the real local end-to-end boundary gap below; the shared-helper seam does not claim that whole path. +The Blaxel exception to ordinary live-owner protection in [Surface process lifecycle](../../../specs/arch/test-runner/test-runner.md#Surface process lifecycle) is proven by [the worker guard seam](blaxel-testshards.md#^blaxel-t-worker-guard) and [automatic recovery acceptance](blaxel-testshards.md#^blaxel-t-recovery-live). The same provider proof carries preservation of the outer shard attempt token through nested surface scopes. These pending assertions do not weaken or relocate the existing ordinary-owner cleanup tests, including fixtures restricted to isolated GitHub execution. + Two test-infrastructure rules concern the product's `tv serve` and are proven where the product is. The recipe for a test that spawns `tv serve` gives the server a temporary [Television home](../../../specs/product/cli.md#^cli-home) whose config file sets port `0`, passes that home with `--home` to the server and every client command, and passes the acquired port as client `--port`. It is a way of writing tests, honored by the design of the tests that spawn the server: the [product CLI proof](../../product/cli.md#Acceptance criteria: CLI spine) declares it for its acceptances, and review checks it elsewhere. No separate assertion is ordered for it; the repository scan below guards only numeric test-service URLs. The daemon-acceptance surface's form of installation (a temporary home, a released stable port written with `tv config set port`, then `tv --home serve --persist`) is carried by the same proof's [home-based persisted-install assertion](../../product/cli.md#^cli-ac-persist-home-install). ## Test hooks diff --git a/proofs/arch/testing-policy.md b/proofs/arch/testing-policy.md index deca7180..3896ca72 100644 --- a/proofs/arch/testing-policy.md +++ b/proofs/arch/testing-policy.md @@ -1,7 +1,27 @@ -*How the testing policy’s development discipline is checked.* +*How worker-owned test builds are proven, and how testing discipline is reviewed.* # Testing policy — proof Proves [specs/arch/testing-policy.md](../../specs/arch/testing-policy.md). -The policy governs authoring and execution discipline, which is checked by review of the contribution and its recorded validation. It orders no tests of its wording or of an agent’s compliance. Runtime enforcement belongs to the test-runner specs and their proofs; its pointer to the shared-branch workflow moves no runtime promise into this spec and orders no additional tests. +## Coverage model + +Authoring and execution discipline is checked by review of the contribution and its recorded validation. It orders no tests of its wording or of an agent's compliance. Runner enforcement belongs to the test-runner proofs. + +Reusable helper builds are runtime behavior owned here. A small contract covers cache publication and failed preparation; real Playwright and Vitest worker lifecycles prove build reuse and final disposal. These are helper tests, not additional product browser walks. Real builder output must be exercised at least once; injected builders earn only lifecycle coverage. + +The built-application default is honored by the setup of the product tests that need it. [Themes](../product/themes-and-appearance.md) and [demo mode](../product/artifacts.md#^af-ac-demo-mode) retain their product assertions while changing their asset-loading boundary. No source-text test asserts that every acceptance uses a particular helper. + +Killed workers cannot promise teardown. The [Blaxel scratch/recovery assertions](test-runner/blaxel-testshards.md#Lease, recovery, and scratch) carry the backstop rather than duplicating a crash experiment here. The new helper assertions below are not yet discharged. + +## Test hooks + +The shared build-lifetime helper accepts a temporary parent and preparation callback; contract tests may supply a builder that writes fixture files, waits on a rendezvous, or rejects, while real callers run the source-CLI or versioned-web preparation. Its teardown entry point is the same one the runner-worker adapters call; exposing it for a contract does not stand in for testing those adapters. + +## Worker-owned builds + +- **Contract — publish successful builds and dispose owned work.** With a real temporary filesystem and recording preparation callbacks, concurrent requests in one worker reuse a pending/successful build for the same key, while distinct version keys remain distinct. A failed preparation removes its partial output and is not published; a later request may prepare it again. Teardown waits for in-flight preparation to settle and removes all owned outputs, including one that failed after creating files. Per-case server disposal cannot invalidate the worker's reusable build. These callbacks mock build/asset preparation and forfeit actual executable output to the worker seam below. This proves [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime) and [cleanup correctness](../../specs/arch/testing-policy.md#Timing, observable status, cleanup, and flakiness). (test to be written) ^testing-t-build-lifetime + +- **Seam — real runner workers own their builds.** The installed Playwright and Vitest runners each execute a small authored fixture through the actual shared helper/adaptor path, using a private temporary root. Two cases assigned to the same worker reuse the source-CLI preparation and a same-version web bundle; a later worker owns a separate lifetime. Source CLI output starts a real product server, and the real web output is served/read, establishing that published results are usable. After per-case server cleanup the shared outputs still exist; after normal worker teardown they do not. No runner, builder, filesystem or server mechanism is mocked. This crosses each framework's teardown boundary under [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime), without a browser or an extra product assertion. [The real version-stamp seam](updates/version-advertisement.md#^t-web-stamp) supplies version breadth; this seam need not rebuild a matrix of versions. (test to be written) ^testing-t-build-worker-seam + +- **Seam — launch preparation failure leaves no private copy.** Through the real product-server helper, a supplied bundle path naming an absent child in a temporary fixture directory makes the real copy fail after allocating the per-launch runtime/home. The filesystem contains no surviving per-launch output, and a subsequent valid launch can use the still-live worker build. No filesystem cleanup or launch mechanism is replaced; the missing bundle path is fixture data. This proves helper cleanup under [Timing, observable status, cleanup, and flakiness](../../specs/arch/testing-policy.md#Timing, observable status, cleanup, and flakiness), while the successful real launch belongs to the worker seam above. (test to be written) ^testing-t-launch-cleanup diff --git a/proofs/arch/themes/delivery.md b/proofs/arch/themes/delivery.md index 96b5843d..551b3752 100644 --- a/proofs/arch/themes/delivery.md +++ b/proofs/arch/themes/delivery.md @@ -6,6 +6,8 @@ Proves [specs/arch/themes/delivery.md](../../../specs/arch/themes/delivery.md). ## Coverage model +The persistence and executable-lifecycle browser walks share the built-application setup declared in [the product coverage model](../../product/themes-and-appearance.md#Coverage model). That setup adaptation remains **(test to be written)**; it changes no delivery outcome and orders no duplicate browser walk here. + Coverage declarations are carried inside the migrated assertion blocks below. The active-route acceptance crosses a really-running server and real package folders for ordinary files, escapes, and validators. JavaScript-route breadth covers all three reserved entries at that same public HTTP boundary: main-entry consent matches and mismatches exact IDs, each manifest declaration is independently enabled, disabled, or absent, and invalid syntax remains fixture bytes rather than being executed. A sibling JavaScript file remains an ordinary static asset. One missing-root recovery walk removes a package with every entry enabled and main consent granted, then proves empty CSS and JavaScript entries, unavailable ordinary assets, and exact restored bytes and validators. The package-root `README.md` exclusion retains its own assertion. Main-script coverage composes a mocked-DOM contract for the resource owner's element mechanics with application-composition contracts for confirmed consent, connection and event wiring, forced reload, and one-shot Settings restoration. DOM, service, history, and navigation mocks forfeit loading, execution, transport, and actual browser reload to [the product executable-theme browser acceptance](../../product/themes-and-appearance.md#^theme-ac-executable-browser) and [reset acceptance](../../product/themes-and-appearance.md#^theme-ac-executable-reset); they buy deterministic lifecycle breadth only. The reset contract deliberately follows installed include presence rather than inferred execution, so it also covers the harmless conservative reload when consent installed an include whose server-side manifest gate returned empty JavaScript. A same-ID manifest refresh remains an in-place include replacement even when it closes that gate. The same product proof carries the required real-Electron execution path. Static ownership and exclusion from standalone entry graphs live in [UI architecture](../ui/index.md#^ui-t-theme-resources). @@ -58,7 +60,7 @@ The [visible application-theme seam](#^theme-delivery-t-visible-app) is covered - **Acceptance** (outer boundary: the built application in Chromium and Firefox against a real server and scratch data directory; fixture: an installed theme importing a nested stylesheet with relative day/night SVG URLs; no mocks or test hooks): selecting the theme resolves its registered `--app-wallpaper-image` relative to the declaring stylesheet, loads the correct asset through the application's composed `--app-wallpaper`, changes the image with appearance, and clears it when selecting None — *(covered by test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme-relative wallpaper URLs render through the built app token in Chromium and Firefox”)*. ^theme-delivery-t-wallpaper-url -- **Acceptance** (outer boundary: a real Chromium browser loads production shell modules and a live-canonical HTML artifact against a really-running server; a selected package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and temporary data directory are fixtures; no mocks or test hooks): changing only the nested stylesheet crosses the real `theme-changed` transport and application-link replacement, then changes imported computed presentation in both document classes without registry refresh — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-delivery-t-active-package-refresh +- **Acceptance** (outer boundary: a real Chromium browser loads the built application and a live-canonical HTML artifact against a really-running server; a selected package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and temporary data directory are fixtures; no mocks or test hooks): changing only the nested stylesheet crosses the real `theme-changed` transport and application-link replacement, then changes imported computed presentation in both document classes without registry refresh — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-delivery-t-active-package-refresh - **Contract** (application theme-resource owner against a mocked DOM, forfeiting browser loading and execution to [the product browser acceptance](../../product/themes-and-appearance.md#^theme-ac-executable-browser)): a refresh creates one cache-busted classic external script in the document head at the connected server's absolute `/theme/main.js`; replacement detaches the prior element before appending the new one, clear removes it, and main-script operations neither replace nor reorder the stylesheet link or sandboxed frames — *(covered by `packages/web/test/theme.test.ts` “installs one cache-busted classic script from the connected server”, “removes the prior script before appending its replacement and clears it”, and “leaves stylesheet and sandboxed frame identity and ordering unchanged”)*. ^theme-delivery-t-app-script-module - **Contract** (renderer composition in a mocked DOM, with connection and application services replaced by controllable boundary mocks, forfeiting real connection event generation and browser resource loading to the browser and Electron assertions below): connection installs the application theme link before display state is ready; a selection change or same-ID resource-refreshing `theme-changed` event replaces it; a consent-membership-only event preserves it; and disconnect removes it — *(covered by test: `packages/web/test/renderer.test.ts` “ties the theme link to connection and theme events before display readiness” and “ties the theme script to confirmed consent, refresh, and connection lifecycle”)*. ^theme-delivery-t-app-link-renderer - **Contract** (application composition against controllable connection and application-service boundaries in a mocked DOM, forfeiting real event transport and script execution to [the product browser acceptance](../../product/themes-and-appearance.md#^theme-ac-executable-browser)): connection leaves the main script absent before confirmed display state; confirmed matching active-ID consent appends the first include only after first-party runtime setup; reconnect reevaluates confirmed state; opt-in injects without replacing the active stylesheet link or frame elements; active-package and registry events with unchanged selection and consent replace the stylesheet and main include; unrelated consent membership preserves all active resource identities; effective appearance changes preserve the stylesheet link and main include while frame replacement follows [the frame-appearance lifecycle](#^theme-delivery-t-frame-appearance-lifecycle); and disconnect removes them. Each path calls the one resource owner rather than using Settings registry state as an execution gate — *(covered by `packages/web/test/renderer.test.ts` “ties the theme script to confirmed consent, refresh, and connection lifecycle” and “reconciles theme frames across confirmed setup, refresh, selection, and connection state”)*. ^theme-delivery-t-app-script-renderer diff --git a/proofs/index.md b/proofs/index.md index 1c3fbb7d..8b6fb083 100644 --- a/proofs/index.md +++ b/proofs/index.md @@ -14,7 +14,7 @@ Generated from the first-line description of every proof by `scripts/specs-index - `licensing.md` (33 lines) — *How Television's licensing promises are proven where users receive them: the license gate over the real tree, the packed CLI tarball, the desktop upload directory, the served browser notices, and Electron's license files in the installed desktop app.* - `tab-pages.md` (30 lines) — *How the promises in Tab pages are proven.* - `telemetry.md` (89 lines) — *How the promises in Telemetry are proven.* - - `themes-and-appearance.md` (55 lines) — *How the promises in Themes and appearance are proven.* + - `themes-and-appearance.md` (57 lines) — *How the promises in Themes and appearance are proven.* - `update-notifications.md` (88 lines) — *How Television's update promises are proven: real browser and Electron journeys for the server-update notice, auto-reload, the desktop self-update notice, the desktop recommendation and the gate, and the desktop product's update check on a candidate build for the desktop app's own updates.* - `versioning.md` (27 lines) — *How Television's release versions are proven where users receive them: the packed CLI package and the desktop upload directory built from the same commit, the built `tv` process and running server, update details, and the native About panel in the desktop product's Mac install check.* - **onboarding/** @@ -28,7 +28,7 @@ Generated from the first-line description of every proof by `scripts/specs-index - `licensing.md` (24 lines) — *How the licensing machinery is proven: generator, gate and loader contracts over authored inputs, and seams through every real shipping build, the suite's inventory handoff and the committed asset manifest.* - `making-skills.md` (18 lines) — *How the promises in Making skills are proven.* - `node-versions.md` (18 lines) — *How the Node version contracts are proven: repository declarations, the advisory floor in the packed CLI package, and the toolchain checker at its two developer entry points.* - - `testing-policy.md` (7 lines) — *How the testing policy’s development discipline is checked.* + - `testing-policy.md` (27 lines) — *How worker-owned test builds are proven, and how testing discipline is reviewed.* - **artifact-frame/** - `artifact-bridge.md` (121 lines) — *How the promises in Artifact bridge are proven.* - `index.md` (24 lines) — *How the promises in Artifact frame (architecture) are proven.* @@ -78,18 +78,18 @@ Generated from the first-line description of every proof by `scripts/specs-index - `sink.md` (58 lines) — *How the promises in Telemetry sink are proven.* - **test-runner/** - `attestation.md` (66 lines) — *How the promises in Test Attestation are proven.* - - `blaxel-testshards.md` (60 lines) — *How the promises in Blaxel Test Shards are proven.* + - `blaxel-testshards.md` (96 lines) — *How the promises in Blaxel Test Shards are proven.* - `flaky-tests.md` (16 lines) — *How the promises in Flaky Tests are proven.* - `github-ci.md` (53 lines) — *How the promises in GitHub Actions CI are proven.* - `preflight.md` (42 lines) — *How the promises in Preflight are proven.* - - `reporting.md` (42 lines) — *How the promises in Reporting are proven.* - - `sharded-execution.md` (52 lines) — *How the promises in Sharded Execution are proven.* + - `reporting.md` (57 lines) — *How the promises in Reporting are proven.* + - `sharded-execution.md` (53 lines) — *How the promises in Sharded Execution are proven.* - `test-registry.md` (38 lines) — *How the promises in Test Registry are proven.* - - `test-runner.md` (75 lines) — *How the promises in Test Runner are proven.* + - `test-runner.md` (77 lines) — *How the promises in Test Runner are proven.* - **themes/** - `authoring.md` (35 lines) — *How theme guidance, generated vocabulary and the authored application reference are proven at the built `television` skill boundary.* - `bundled-installation.md` (42 lines) — *How the promises in Bundled theme installation are proven.* - - `delivery.md` (87 lines) — *How the promises in Theme delivery are proven.* + - `delivery.md` (89 lines) — *How the promises in Theme delivery are proven.* - `index.md` (63 lines) — *How the promises in Theme architecture are proven.* - **ui/** - `conformance.md` (15 lines) — *How production UI conformance is checked while automated markup comparison is scheduled for TV-649.* diff --git a/proofs/product/artifacts.md b/proofs/product/artifacts.md index e402e310..6b1df013 100644 --- a/proofs/product/artifacts.md +++ b/proofs/product/artifacts.md @@ -6,7 +6,7 @@ Proves [specs/product/artifacts.md](../../specs/product/artifacts.md). ## Coverage model -Coverage declarations are carried inside the migrated assertion blocks below. +Coverage declarations are carried inside the migrated assertion blocks below. The two demo-mode acceptances and the bridge non-participation seam share the built application and built unsupported-URL view from the real product-server helper. Their setup adaptation is **(test to be written)**; the existing citations name the walks to adapt without adding product assertions or a second browser pass. ## Test hooks @@ -49,12 +49,12 @@ Coverage relationship: serving an artifact's content at `/artifact//` to a b Coverage relationship: fresh loading after a channel switch is crossed by [arch/artifact-frame/reload-navigation.md#^rn-ac-persist-browser](../arch/artifact-frame/reload-navigation.md#^rn-ac-persist-browser), whose real-browser case removes every frame while another channel is current and observes a newly loaded frame on return. Its history restoration is reload-navigation's promise; this spec adds no second channel-switch walk. Rearrangement continuity is deliberately not promised, so it creates no assertion. -Browser demo mode ([#^af-demo-mode](../../specs/product/artifacts.md#^af-demo-mode)) is proven by two Chromium acceptance walks, with a routing contract for the breadth of which artifacts it reaches. Shared declaration for the two walks: the outer boundary is Chromium driving the production app against a really-running server launched with the server-home hook above; the external page is a two-page HTML fixture served by a local HTTP server on an origin other than the Television server's, never a public site; the external page artifact is created through the production API; the default motion override is the only mock. Playwright reads the cross-origin frame's document to observe which fixture page is shown. +Browser demo mode ([#^af-demo-mode](../../specs/product/artifacts.md#^af-demo-mode)) is proven by two Chromium acceptance walks, with a routing contract for the breadth of which artifacts it reaches. Shared declaration for the two walks: the outer boundary is Chromium driving the built application and built unsupported-URL view against a really-running server launched with the server-home hook above; the external page is a two-page HTML fixture served by a local HTTP server on an origin other than the Television server's, never a public site; the external page artifact is created through the production API; the default motion override is the only mock. Playwright reads the cross-origin frame's document to observe which fixture page is shown. - **Acceptance** (shared demo-mode walk): with no marker file, the external page artifact's frame shows the unsupported-URL error page. After the marker file is created and the app is reloaded, the frame's document is the fixture's first page at its own URL, the frame shows no Back or Forward control, and the artifact icon is a button named by [content.yml#return_aria_label](../../specs/ui/app/artifact-frame/content.yml#return_aria_label). After the marker file is deleted and the app is reloaded again, the frame shows the unsupported-URL error page and the icon is not a button — *(covered by test: `packages/web/test/e2e/browser-demo-mode.test.ts` "the home-directory marker turns direct external pages on and off at app load (^af-ac-demo-mode)")*. ^af-ac-demo-mode - **Acceptance** (shared demo-mode walk, marker present): following the fixture's link shows its second page; pressing the artifact icon shows the first page again. Following the link again and reloading the app shows the first page. Following the link once more, switching to another channel, and returning shows the first page. Back and Forward remain absent throughout — *(covered by test: `packages/web/test/e2e/browser-demo-mode.test.ts` "the artifact icon, an app reload, and a channel round trip each return to the original page (^af-ac-demo-return)")*. ^af-ac-demo-return - **Contract** (the production artifact-frame view under jsdom with an authored external page artifact and a fake application service posing demo mode on — mocks that forfeit real document loading to [#^af-ac-demo-return](#^af-ac-demo-return); theme-change and artifact-content-change events are injected on that service): neither event reassigns the demo page's frame `src`, which stays the artifact's own URL — *(covered by test: `packages/web/test/artifact-view.test.ts` "browser demo mode artifact frame > leaves a demo-mode page's URL untouched through theme and content change events (^af-ac-demo-untouched)")*. ^af-ac-demo-untouched - **Contract** (the production routing functions `artifactRenderRoute` and `artifactDocumentRoute` over authored artifact records and recorded current URLs; no mocks): with demo mode on in a browser, `http:` and `https:` artifact URLs that are not Television artifact addresses route to a direct iframe at the artifact's own URL, including a URL on the Television host itself; a Television artifact address keeps the shared-artifact route both on another host and on the connected Television host; a non-web URL keeps the unsupported-URL page. Path and Markdown artifacts, a Television-served artifact whose recorded current URL is an external web page, and every artifact in the desktop app route identically with demo mode on and off — *(covered by tests: `packages/web/src/artifact-dispatcher.test.ts` "browser demo mode routing", all four cases)*. ^af-ac-demo-scope -Coverage relationship: the two walks cross the real server's marker-file check, its delivery to the browser, routing, and frame rendering, and compose with [#^af-ac-demo-scope](#^af-ac-demo-scope), which carries the classification and routing breadth off the walked path, and [#^af-ac-demo-untouched](#^af-ac-demo-untouched), which keeps Television's own reload triggers away from the page. The walks reach the server through the development proxy, so app and server share an origin; that the flag route sends no CORS headers is [artifact isolation's route contract](../arch/artifact-frame/isolation.md#^iso-t-routes). That demo pages cannot forward the navigation chord or feed theme pointer effects is carried by the bridge's non-participation assertions [arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-seam](../arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-seam) and [arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-host](../arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-host). A site that refuses framing is outside Television's behavior, since Television only leaves the browser's own outcome in place, and orders no test. Which views show the return button, and what pressing it requests, are the frame's contracts at [ui/app/artifact-frame/index.md#^af-ui-ac-return-scope](../ui/app/artifact-frame/index.md#^af-ui-ac-return-scope) and [ui/app/artifact-frame/index.md#^af-ui-ac-frame-actions](../ui/app/artifact-frame/index.md#^af-ui-ac-frame-actions). +Coverage relationship: the two walks cross the real server's marker-file check, its delivery to the browser, routing, and frame rendering, and compose with [#^af-ac-demo-scope](#^af-ac-demo-scope), which carries the classification and routing breadth off the walked path, and [#^af-ac-demo-untouched](#^af-ac-demo-untouched), which keeps Television's own reload triggers away from the page. The built application and server share an origin; that the flag route sends no CORS headers is [artifact isolation's route contract](../arch/artifact-frame/isolation.md#^iso-t-routes). That demo pages cannot forward the navigation chord or feed theme pointer effects is carried by the bridge's non-participation assertions [arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-seam](../arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-seam) and [arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-host](../arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-host). A site that refuses framing is outside Television's behavior, since Television only leaves the browser's own outcome in place, and orders no test. Which views show the return button, and what pressing it requests, are the frame's contracts at [ui/app/artifact-frame/index.md#^af-ui-ac-return-scope](../ui/app/artifact-frame/index.md#^af-ui-ac-return-scope) and [ui/app/artifact-frame/index.md#^af-ui-ac-frame-actions](../ui/app/artifact-frame/index.md#^af-ui-ac-frame-actions). diff --git a/proofs/product/themes-and-appearance.md b/proofs/product/themes-and-appearance.md index 210a5ae0..ff2fdff2 100644 --- a/proofs/product/themes-and-appearance.md +++ b/proofs/product/themes-and-appearance.md @@ -6,6 +6,8 @@ Proves [specs/product/themes-and-appearance.md](../../specs/product/themes-and-a ## Coverage model +The persistence and executable-lifecycle walks in `theme-product-acceptance.test.ts` load the built application through the real stable-front proxy, keeping the browser origin and authentication across backends that bind port `0` on restart. Their built-application setup adaptation is **(test to be written)**; the citations below identify the existing walks to adapt, not completed evidence of that new boundary. Their product outcomes, fixtures and motion declarations stay the same. The separate `settings.test.ts` registry/control walk keeps its existing development-module boundary. Reset already uses the built application and is unchanged. + Coverage declarations are carried inside the migrated assertion blocks below. The executable-theme browser spine uses two connected application documents in one real Chromium session against a running production server. Real package fixtures drive main-enabled, unflagged, iframe-enabled, syntax-error, and uncaught-exception paths through Settings, persisted display state, the three public script routes, the production application resource owner, two real iframe boundaries, and an artifact document. Theme-owned markers, execution counters, package assets, and frame messages make absence before main consent, main execution in both application documents, automatic sandbox execution, repeat execution, document-lifetime effects, reload reset, classic-script-relative URL resolution, and frame replacement observable without a test hook or mock. Browser error events and ordinary application controls establish failure isolation. @@ -39,13 +41,13 @@ Clouds' association with Television 1.3 composes with [product versioning's rele ### Acceptance criteria -- **Acceptance** (outer boundaries: real Settings controls over HTTP and websocket connections; the persistence matrix loads production shell modules through the test Vite server against a built server, while the cross-browser `None`/Clouds case loads that server's built shell; valid, invalid, and dot-prefixed theme folders and real temporary homes are fixtures; no mocks or test hooks): a fresh installation offers None plus the themes in [the bundled inventory](../../specs/ui/themes/bundled.yml); refresh shows valid packages and readable invalid-folder errors while every dot-prefixed folder is absent from options and errors; selection propagates to a second client; `None` selects the null theme and clears the active theme ID; the selected theme renders in Chromium and Firefox; and the server-confirmed selection survives a server restart and reconnection in the same browser page — *(covered by `packages/web/test/e2e/settings.test.ts` “settings controls cross the registry and confirmed display state” and `packages/web/test/e2e/theme-product-acceptance.test.ts` “the real dropdown renders its selected built-shell theme in Chromium and Firefox” and “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-ac-selection +- **Acceptance** (outer boundaries: real Settings controls over HTTP and websocket connections; the persistence and cross-browser `None`/Clouds walks load the built application, while the separate registry/control walk in `settings.test.ts` retains its development-module boundary; valid, invalid, and dot-prefixed theme folders and real temporary homes are fixtures; no mocks or test hooks): a fresh installation offers None plus the themes in [the bundled inventory](../../specs/ui/themes/bundled.yml); refresh shows valid packages and readable invalid-folder errors while every dot-prefixed folder is absent from options and errors; selection propagates to a second client; `None` selects the null theme and clears the active theme ID; the selected theme renders in Chromium and Firefox; and the server-confirmed selection survives a server restart and reconnection in the same browser page — *(covered by `packages/web/test/e2e/settings.test.ts` “settings controls cross the registry and confirmed display state” and `packages/web/test/e2e/theme-product-acceptance.test.ts` “the real dropdown renders its selected built-shell theme in Chromium and Firefox” and “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-ac-selection - **Acceptance** (outer boundary: Firefox drives the real Settings controls in the built application against a really-running production server and its real event transport; fixed-dark and light theme packages, a ready Markdown editor with authored contents, and a temporary home are fixtures; no mocks or test hooks): selecting the light theme changes both the shell and the same loaded Markdown editor from the fixed-dark presentation to the light presentation without replacing the editor document or losing its contents — *(covered by test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “Firefox refreshes a ready Markdown editor from a fixed-dark theme to light”)*. The same browser walk supplies the delivery evidence at [theme-delivery-t-markdown-refresh-firefox](../arch/themes/delivery.md#^theme-delivery-t-markdown-refresh-firefox). ^theme-ac-markdown-refresh-firefox - **Acceptance** (outer boundary: two real Chromium application documents drive the built application's Settings controls against a running production server and its real event transport; main-enabled, unflagged, syntax-error, and uncaught-exception theme packages, a package-relative asset, an artifact document, and a temporary home are fixtures; no mocks or test hooks): before consent, the selected `enableMainJS: true` theme has no `main.js` include or request in either application document. Granting exact-ID consent runs its classic main script in both documents after the application runtime is available, exposes `document.currentScript.src`, resolves the package asset from that URL, survives a built-server restart, and leaves the artifact script-free. Appearance changes do not rerun it. An unrelated active-package edit, restoration after complete package-root loss, and server reconnection each rerun it; the missing-root interval executes nothing. An unflagged `main.js` executes nowhere. Separately consented syntax-error and uncaught-exception main scripts report browser errors and leave an ordinary Settings control operational — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-executable-browser - **Acceptance** (outer boundary: the same two real Chromium application documents and running production server; a main-enabled theme whose script installs document-lifetime probes, a second theme, and persisted Settings-open state are fixtures; no mocks or test hooks): while the main include is active, withdrawing consent, selecting the second theme, and selecting `None` each reload both connected application documents, clear the prior script's document-lifetime effects, and start under the confirmed destination state. Only a document whose Settings popover was open reopens it; its URL marker is consumed before reopening and causes no later reload. After selecting the second theme, selecting the still-consented main-enabled theme again reinstalls its include and executes it without another consent grant. Changing themes while no main include is installed remains in place — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable theme reset reloads connected Chromium documents under confirmed state”)*. ^theme-ac-executable-reset - **Acceptance** (outer boundary: the same real Chromium application and running production server; a package independently enabling both sandbox entries, a destination theme, syntax-error and uncaught-exception iframe entries, Settings and application modal-dialog controls, and pointer-observation probes are fixtures; no mocks or test hooks): without main-page consent, the background and foreground entries execute in separate opaque-origin `allow-scripts` frames at their specified layers and cannot access the application DOM. Theme selection and an unrelated active-package edit replace both frame documents; package-root loss replaces them with empty entries, restoration executes fresh frames, and registry refresh that disables either declaration removes only that frame. Syntax and uncaught runtime errors stay in their affected frames while the application operates. Trusted pointer move, down, up, cancel, and click events in the application document reach both frames with viewport coordinates and button state while the same underlying application sequence, including click, remains effective; a drag that produces no click forwards its down, movement, and up but no click. Pointer move, down, up, and click inside the bridge-capable artifact reach both frames in application-viewport coordinates while the artifact's own click remains effective. Unexpected focus returns first to the last connected application element and then to `#app` when that element is unavailable. The foreground layers do not prevent using the production Settings popover in its document layer or an application modal dialog in the browser top layer — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-sandboxed-frames - **Acceptance** (outer boundary: the production Settings appearance control in a real Chromium application against a running production server; a real active package enabling both sandbox entries, entry-script startup probes, visible application probes behind transparent regions of both frames, and a temporary home are fixtures; no mocks or test hooks): under the [frame lifecycle](../../specs/product/themes-and-appearance.md#^theme-iframe-lifecycle), an effective light-to-dark appearance change removes both old frame documents before installing replacements, leaves no old-appearance frame, and reruns both entries in documents whose root `data-theme` is `dark`. Each replacement host and document root has a used dark scheme, and the application probes remain visibly unchanged through the frames' transparent regions — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-frame-appearance -- **Acceptance** (outer boundary: a real Chromium browser loads production shell modules and a live-canonical HTML artifact against a running built server and its real `/events` transport; a real active package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and a temporary home are fixtures; no mocks or test hooks): changing only the nested stylesheet updates the application and artifact presentation; moving the complete active package away produces their foundation and canonical baselines while the exact selected theme ID remains confirmed; restoring it at the same path restores both presentations and keeps that ID, without registry refresh or another selection — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-ac-active-package +- **Acceptance** (outer boundary: a real Chromium browser loads the built application and a live-canonical HTML artifact against a running built server and its real `/events` transport; a real active package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and a temporary home are fixtures; no mocks or test hooks): changing only the nested stylesheet updates the application and artifact presentation; moving the complete active package away produces their foundation and canonical baselines while the exact selected theme ID remains confirmed; restoring it at the same path restores both presentations and keeps that ID, without registry refresh or another selection — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-ac-active-package - **Acceptance** (outer boundary: the real Electron app and another client connected to a running server; a theme enabling main, background, and foreground entries, a second theme, an artifact webview, and an isolated Electron profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): before consent the main script does not execute, while both sandboxed frames are created and execute in the application. Pointer move, down, up, and click inside the ordinary HTML artifact webview reach both theme frames in application-viewport coordinates while the artifact's click remains effective. Granting consent through Settings executes `main.js` in the application renderer but no theme JavaScript in the artifact webview, and the renderer exposes the same Node-facing globals and preload bridge before and after activation. Consent survives application restart; remote main opt-out and remote selection of the second theme each reload the application document, clear its document-lifetime probe, and apply confirmed state. A remote same-theme active-package event replaces both frame documents without itself causing a full application reload — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “executes an executable theme in Electron and resets on remote consent and selection changes”)*. ^theme-ac-executable-electron - **Acceptance** (outer boundary: the real Electron app connected to a running server; an installed theme and isolated Electron profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): the selected application theme renders again after quitting and relaunching through the saved connection — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “renders the persisted app theme after quitting and relaunching”)*. ^theme-ac-selection-electron - **Acceptance** (outer boundary: the built application in a real Chromium browser against a really-running server; one adaptive package and one fixed-dark package, local Television-managed HTML and markdown artifacts, a third-party URL artifact, both sandboxed entries with startup probes, dedicated computed-style probes, controlled system appearance, and a real temporary home are fixtures; no mocks or test hooks): under the adaptive theme, `Adapt to system`, `Light`, and `Dark` produce their expected effective appearance in the application and applicable artifacts. Under the fixed-dark theme, selecting and persisting an opposing Light preference leaves the application root, foundation mode values and native scheme, both artifact classes, and both frame host/document pairs dark; it does not reload app or artifacts, and it keeps the existing frame documents because effective appearance did not change. The same fixed results survive a built-server restart. Returning to the adaptive theme applies the latest Light preference; that effective change replaces and reruns both frames while leaving their transparent composition intact — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart” and “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-appearance diff --git a/specs/arch/test-runner/blaxel-testshards.md b/specs/arch/test-runner/blaxel-testshards.md index 4c983b16..8b933ef9 100644 --- a/specs/arch/test-runner/blaxel-testshards.md +++ b/specs/arch/test-runner/blaxel-testshards.md @@ -36,54 +36,54 @@ Fresh workers receive the labels that make them discoverable for test dispatch o The coordinator leases one worker per shard through `/tmp/tv-testshard.lock`. Its `owner.json` records the unique `lockId`, run/shard/attempt identity, owner and acquisition time, worker-local last renewal and `recoverAfter` deadline, and dispatch ownership. Acquisition, renewal, dispatch registration, recovery, and release serialize their worker-local changes under one short OS-released guard outside the disposable lock directory. Rechecking identity under that guard prevents an earlier renewal or release from overwriting a new owner. The guard is not held during network downloads. ^blaxel-reservation -Normal runs use at most 32 concurrent lease acquisitions (`MAX_PARALLEL_LEASE_ACQUISITIONS`), starting another as each settles. Each attempt retains its unique shard lock ID, validates `uname -m` and `process.arch`, and records a lease as soon as it succeeds. The coordinator waits for every requested lease before starting any shard process. Discovery failure, an undersized deployed pool, or incomplete acquisition starts no shard. The attempt records one infrastructure-failed result per requested shard, including whether a lease was acquired, `executionStarted: false`, the assignment, and the exact cause. Partial leases follow the release conditions below; incomplete shards remain eligible for infrastructure retry. +Normal runs use at most 32 concurrent lease acquisitions (`MAX_PARALLEL_LEASE_ACQUISITIONS`), starting another as each settles. Each shard attempt retains its unique lock ID, validates `uname -m` and `process.arch`, and records a lease as soon as it succeeds. The coordinator waits for every requested lease before starting any shard process. Discovery failure, an undersized deployed pool, or incomplete acquisition starts no shard. The attempt records one infrastructure-failed result per requested shard, including whether a lease was acquired, `executionStarted: false`, the assignment, and the exact cause. Partial leases follow the release conditions below; incomplete shards remain eligible for infrastructure retry. -Only the coordinator renews a reservation, throughout acquisition barriers, preparation, execution, teardown, and result download. Each successful renewal sets `recoverAfter` to worker-local time plus the recovery interval: 20 minutes by default, overridden by `--lock-ttl-seconds`. This interval is independent of the shard execution timeout. Renewal requires the same owner and an unexpired deadline; it cannot revive an expired reservation. A surviving shard cannot renew on its coordinator's behalf. +Only the coordinator renews a lease, throughout acquisition barriers, preparation, execution, teardown, and result download. Each successful renewal sets `recoverAfter` to worker-local time plus the recovery interval: 20 minutes by default, overridden by `--lock-ttl-seconds`. This interval is independent of the shard execution timeout. Renewal requires the same owner and an unexpired deadline; it cannot revive an expired lease. A surviving shard cannot renew on its coordinator's behalf. -The outer shard has a unique attempt owner token inherited by setup commands, surface supervisors, and their descendants. It remains available alongside the surface token that nested supervisors replace. Dispatch is recorded before the API request; the remote entry point checks the matching, unexpired reservation and registers its process name and PID/start identity under the guard before touching checkout or starting setup. A delayed request cannot launch obsolete work after recovery. Uploads use attempt-specific paths. Unknown dispatch outcomes retain the reservation until resolved or recovered. +The outer shard has a unique shard attempt owner token inherited by setup commands, surface supervisors, and their descendants. It remains available alongside the surface token that nested supervisors replace. Dispatch is recorded before the API request; the remote entry point checks the matching, unexpired lease and registers its process name and PID/start identity under the guard before touching checkout or starting setup. A delayed request cannot launch obsolete work after recovery. Uploads use paths specific to the shard attempt. Unknown dispatch outcomes retain the lease until resolved or recovered. -Normal release requires confirmed process teardown and completed retrieval of that attempt's reports. The shard shell finalizes evidence but never unlocks on exit. Named Blaxel process stop or successful shell completion does not establish that descendants are gone: audit the attributed processes and use token- and PID/start-validated TERM/KILL cleanup for survivors. Preserve the audit in the shard log. Remote cleanup activates nvm before enabling nounset, with strict handling in the cleanup body. An unreadable audit, uncertain dispatch, or surviving owner retains the reservation and excludes the worker from immediate test retries. ^blaxel-release +Normal release requires confirmed process teardown and completed retrieval of that shard attempt's reports. The shard shell finalizes evidence but never unlocks on exit. Named Blaxel process stop or successful shell completion does not establish that descendants are gone: audit the attributed processes and use token- and PID/start-validated TERM/KILL cleanup for survivors. Preserve the audit in the shard log. Remote cleanup activates nvm before enabling nounset, with strict handling in the cleanup body. An unreadable audit, uncertain dispatch, or surviving owner retains the lease and excludes the worker from immediate test retries. ^blaxel-release -While still renewing and holding the lock, download summaries, native reports, logs, and setup/cleanup/diagnostic artifacts; check their identities and required evidence. Missing or corrupt evidence remains an infrastructure failure. An early setup failure records which outputs were never produced and retains those that were. Incomplete retrieval keeps the reservation while retrieval is retried, without rerunning tests. An unused lease can be released only when execution was never requested, pending writes have settled, and any produced evidence has been retained. The fixed worker results tree must not be reset by a later checkout before this boundary. +While still renewing and holding the lock, download summaries, native reports, logs, and setup/cleanup/diagnostic artifacts; check their identities and required evidence. Missing or corrupt evidence remains an infrastructure failure. An early setup failure records which outputs were never produced and retains those that were. Incomplete retrieval keeps the lease while retrieval is retried, without rerunning tests. An unused lease can be released only when execution was never requested, pending writes have settled, and any produced evidence has been retained. The fixed worker results tree must not be reset by a later checkout before this boundary. -On `SIGINT` or `SIGTERM` while an attempt exists, the attempt becomes interrupted even if its shard promises have all just completed. Completed shard results remain intact. An unfinished shard retains downloaded summaries, native results, task artifacts, parsed task evidence, assignments, collections, and logs, and records `interrupted/coordinator-interruption` plus its known lease and execution state. A pending shard has no synthetic task result. A dispatched process whose start is unconfirmed omits `executionStarted`; absence means unknown, while `false` means execution was not requested. The coordinator makes this interrupted snapshot authoritative before cleanup and retrieval, and cleanup cannot replace it with an ordinary completion report. A report-write failure does not suppress process cleanup. The first signal exits 130 after cleanup; a second exits immediately. Any reservation that cannot be released safely stops renewing when the coordinator exits and follows abandoned-run recovery. A signal before the first attempt has no shard result to preserve; the canonical runner treats absent provider evidence as incomplete. Console summaries name only retained log files. +On `SIGINT` or `SIGTERM` while an attempt exists, the attempt becomes interrupted even if its shard promises have all just completed. Completed shard results remain intact. An unfinished shard retains downloaded summaries, native results, task artifacts, parsed task evidence, assignments, collections, and logs, and records `interrupted/coordinator-interruption` plus its known lease and execution state. A pending shard has no synthetic task result. A dispatched process whose start is unconfirmed omits `executionStarted`; absence means unknown, while `false` means execution was not requested. The coordinator makes this interrupted snapshot authoritative before cleanup and retrieval, and cleanup cannot replace it with an ordinary completion report. A report-write failure does not suppress process cleanup. The first signal exits 130 after cleanup; a second exits immediately. Any lease that cannot be released safely stops renewing when the coordinator exits and follows abandoned-run recovery. A signal before the first attempt has no shard result to preserve; the canonical runner treats absent provider evidence as incomplete. Console summaries name only retained log files. ### Abandoned-run recovery -After renewal stops for the recovery interval, the next ordinary acquisition stops and audits the abandoned attempt's owned processes, clears its leftovers, and takes the lock. The same sequence applies to a quiet worker and to a surviving supervisor or test descendant. Expiry authorizes cleanup, never reuse beneath live processes. This operational definition of abandonment also covers a coordinator suspended or disconnected longer than the interval; it does not claim to prove the coordinator's OS process died. Quiet abandoned workers wait the same interval to keep recovery to one procedure. ^blaxel-recovery +After renewal stops for the recovery interval, the next ordinary acquisition stops and audits the abandoned shard attempt's owned processes, clears its leftovers, and takes the lock. The same sequence applies to a quiet worker and to a surviving supervisor or test descendant. Expiry authorizes cleanup, never reuse beneath live processes. This operational definition of abandonment also covers a coordinator suspended or disconnected longer than the interval; it does not claim to prove the coordinator's OS process died. Quiet abandoned workers wait the same interval to keep recovery to one procedure. ^blaxel-recovery -Under the guard, recheck the old identity and deadline, terminate only its attributed processes with revalidated identity, and confirm the worker has no remaining live or stale test owners. This recovery may stop the abandoned attempt's still-live supervisors; ordinary stale-owner cleanup continues to protect other live owners. Only then remove abandoned scratch and the known leftovers below, discard that attempt's undownloaded results, and publish the new reservation. A bounded [recovery receipt](./reporting.md#Resource diagnostics) identifies the old run, deadline, process cleanup, and discarded reports; it survives the results reset as part of the incoming attempt's evidence. This is the explicit exception to normal download-before-release. Lost evidence can never establish a pass for the abandoned run. +Under the guard, recheck the old identity and deadline, terminate only its attributed processes with revalidated identity, and confirm the worker has no remaining live or stale test owners. This recovery may stop the abandoned shard attempt's still-live supervisors; ordinary stale-owner cleanup continues to protect other live owners. Only then remove abandoned scratch and the known leftovers below, discard that shard attempt's undownloaded results, and publish the new lease. A bounded [recovery receipt](./reporting.md#Resource diagnostics) identifies the old run, deadline, process cleanup, and discarded reports; it survives the results reset as part of the incoming shard attempt's evidence. This is the explicit exception to normal download-before-release. Lost evidence can never establish a pass for the abandoned run. -Interrupted recovery leaves the old expired identity until another acquisition completes cleanup. A returning coordinator cannot renew, release, dispatch, or mutate an attempt whose ownership it lost; it reports infrastructure failure. Failure to identify or stop owners holds and reports the worker instead of dispatching tests. Missing or corrupt ownership evidence cannot authorize signalling by name. An acquisition interrupted before complete registration can be reclaimed after a clean worker audit, since execution cannot precede registration. Ordinary attributable coordinator crashes recover automatically; persistent cleanup/provider failures require operator action. +Interrupted recovery leaves the old expired identity until another acquisition completes cleanup. A returning coordinator cannot renew, release, dispatch, or mutate a shard attempt whose ownership it lost; it reports infrastructure failure. Failure to identify or stop owners holds and reports the worker instead of dispatching tests. An unreadable or incomplete lock record may be reclaimed by ordinary acquisition or pool deletion only after a clean worker audit; otherwise hold and report it. Execution cannot precede complete registration, and missing ownership evidence cannot authorize signalling by name. Ordinary attributable coordinator crashes recover automatically; persistent cleanup/provider failures require operator action. ### Older runners and pool operations -Write `expiresAt: 0` in new records: older runners reclaim only positive past timestamps, so they leave these reservations alone. For a legacy record with no renewable deadline, respect its unexpired TTL; after expiry, require a clean worker audit before acquisition. Skip and report live or ambiguous legacy work. Its normal exit trap removes only its matching `lockId`. Updated runners cannot protect an older coordinator's downloads after that coordinator releases early; branches acquire the corrected behavior when they integrate it. No pool-wide drain or simultaneous branch update is required. +**Transitional, while branches still run the timestamp-only lock protocol:** write `expiresAt: 0` in new records: older runners reclaim only positive past timestamps, so they leave these leases alone. For a legacy record with no renewable deadline, respect its unexpired TTL; after expiry, require a clean worker audit before acquisition. Skip and report live or ambiguous legacy work. Its normal exit trap removes only its matching `lockId`. Updated runners cannot protect an older coordinator's downloads after that coordinator releases early; branches acquire the corrected behavior when they integrate it. -Pool deletion checks the same identity and deadline under the guard and refuses unexpired reservations. An unlocked worker may be reserved for deletion after a clean owner audit. Explicit deletion may also reserve and destroy a worker with an expired reservation even when process cleanup failed, because sandbox deletion is the cleanup. Before issuing deletion, remove the worker from test dispatch eligibility; failed or delayed deletion must not admit a new test run. The manager continues to identify and report it as unavailable for operator repair. An unreadable ownership record is not proof of expiry. Read-only pool listing needs no reservation. Fresh provisioning follows [Pool](#Pool). +Pool deletion checks the same identity and deadline under the guard and refuses unexpired leases. An unlocked worker may be reserved for deletion after a clean owner audit. Explicit deletion may also reserve and destroy a worker with an expired lease even when process cleanup failed, because sandbox deletion is the cleanup. Before issuing deletion, remove the worker from test dispatch eligibility; failed or delayed deletion must not admit a new test run. The manager continues to identify and report it as unavailable for operator repair. Unreadable records follow the recovery rule above. A newly created worker that has never been published for test dispatch may be deleted without a lease. Any other worker that cannot answer the ownership check is held for operator repair; discovery ineligibility alone does not establish that an earlier run has stopped. Fresh provisioning follows [Pool](#Pool). -`--audit-stale-owners` reserves an available worker without recovering a held reservation or deleting residue before observing it. It holds its reservation through its prerequisite setup and report collection, then releases normally. Held workers are reported/skipped. This preserves the diagnostic's read-only view of stale owners. +`--audit-stale-owners` reserves an available worker without recovering a held lease or deleting residue before observing it. It holds its lease through its prerequisite setup and report collection, then releases normally. Held workers are reported/skipped. This preserves the diagnostic's read-only view of stale owners. ## Temporary files -Every leased shard attempt has a fresh private directory under `/tmp/tv-testshards/`, identified with its run, shard, attempt, and lock. An absolute `TMPDIR` points to its `tmp/` child before preparation or tests begin and is preserved by child environments. Uploaded scripts, plans, askpass helpers, and setup sidecars use the attempt directory; finalized reports live outside it. Paths stay short enough for browser Unix sockets. Intentional npm, Electron, browser-download, and dependency-hash caches retain their existing persistent locations. ^blaxel-scratch +Every leased shard attempt has a fresh private directory under `/tmp/tv-testshards/`, identified with its run, shard, provider attempt, and lock. An absolute `TMPDIR` points to its `tmp/` child before preparation or tests begin and is preserved by child environments. Uploaded scripts, plans, askpass helpers, and setup sidecars are written in the shard attempt directory. Setup sidecars are finalized into the results tree before that directory is removed. Paths stay short enough for browser Unix sockets. Intentional npm, Electron, browser-download, and dependency-hash caches retain their existing persistent locations. ^blaxel-scratch -After taking the reservation and confirming no prior execution remains, each run removes only: +After taking the lease and confirming no prior execution remains, each run removes only: - Real directories directly under `/tmp` matching `television-source-cli-e2e-*` or `television-web-bundle-*`. -- Every non-current attempt directory directly under `/tmp/tv-testshards/`, including ones with missing or incomplete metadata. +- Every non-current shard attempt directory directly under `/tmp/tv-testshards/`, including ones with missing or incomplete metadata. -The lock and clean worker audit establish disuse; metadata or age alone does not. Prior result retrieval is protected by the reservation, with only the abandoned-run exception above. Reject symlink roots, unexpected mount boundaries, and paths escaping the allowed parent; recursive removal must not follow links into another tree. Preserve the current attempt, lock and guard, provider uploads, X11 state, unrelated temporary files, checkouts, and persistent caches. Report counts and allocated bytes removed per prefix, filesystem free space before/after, and skips/errors. Do not equate allocated bytes removed with free-space change or claim a partial deletion reclaimed an entire directory. ^blaxel-known-leftovers +The lock and clean worker audit establish disuse; metadata or age alone does not. Prior result retrieval is protected by the lease, with only the abandoned-run exception above. Reject symlink roots, unexpected mount boundaries, and paths escaping the allowed parent; recursive removal must not follow links into another tree. Preserve the current shard attempt, lock and guard, provider uploads, X11 state, unrelated temporary files, checkouts, and persistent caches. Report counts and allocated bytes removed per prefix, filesystem free space before/after, and skips/errors. ^blaxel-known-leftovers -After success, failure, timeout, or interruption, stop attributed processes, finalize evidence outside scratch, and remove the current attempt directory while retaining the lock through download. An uncertain teardown preserves scratch. Failed cleanup retains its cause and native outcomes, but prevents new test dispatch until cleanup succeeds; an inconclusive worker audit likewise skips deletion and dispatch. Download available evidence even when cleanup fails. Directory deletion cannot release Chromium's open, unlinked shared-memory files: their owning processes must stop first. +After success, failure, timeout, or interruption, stop attributed processes, finalize evidence outside scratch, and remove the current shard attempt directory while retaining the lock through download. An uncertain teardown preserves scratch. Failed cleanup retains its cause and native outcomes, but prevents new test dispatch until cleanup succeeds; an inconclusive worker audit likewise skips deletion and dispatch. Download available evidence even when cleanup fails. Directory deletion cannot release Chromium's open, unlinked shared-memory files: their owning processes must stop first. -Retain at most the current attempt's bounded resource evidence on the worker under the existing `.testshards/results` tree. The next reset follows protected download, or the explicitly recorded abandonment decision above. There is no accumulating cross-run diagnostic archive. Coordinator-side run retention is unchanged. Evidence contents and byte limits belong to [reporting.md](./reporting.md#Resource diagnostics). +Retain at most the current shard attempt's bounded resource evidence on the worker under the existing `.testshards/results` tree. The next reset follows protected download, or the explicitly recorded abandonment decision above. There is no accumulating cross-run diagnostic archive. Coordinator-side run retention is unchanged. Evidence contents and byte limits belong to [reporting.md](./reporting.md#Resource diagnostics). ## Shard lifecycle After acquiring the complete batch, the coordinator uploads each shard's plan and worker script before starting its remote process. A rejected upload returns an infrastructure-failed shard result with the sandbox, assignment, and `transport/worker-dispatch` cause; it cannot reject the whole attempt without a report. -Each shard runs a script on its worker whose steps are recorded as setup timings: `checkout` (clone if the worker has no checkout, point `origin` at the run's repository, then fetch and hard-reset to the commit), `runtime-activation`, `runtime-versions`, `deps`, `system-deps` (`xauth`, `libgtk-3-0`, and `iproute2` for `ss`), `owner-lifecycle`, `playwright-cache`, `electron-repair`, then `test-run`. The `electron-repair` phase invokes the installed Electron package's own `install.js` with `/cache/electron` as its archive cache. Electron's installer reuses a complete runtime or prepares an absent one; the phase carries no separate download/extraction implementation and does not delete runtime files. ^blaxel-electron-repair After checkout, `runtime-activation` loads the provisioned nvm and runs `nvm use` from the checked-out `.nvmrc`; it does not run a second version checker or install/repair a runtime. All dependency, repair, and test commands inherit the activated Node/npm path. The `system-deps` step asserts these tools on every lease and installs the package set when any prerequisite is absent. `owner-lifecycle` leaves live-supervisor tokens untouched, reaps stale token-bearing groups with revalidated TERM/KILL, and fails before `test-run` whenever it found contamination. That sandbox is excluded from every later attempt by the coordinator. A configured infrastructure retry may run the shard on a different sandbox, whose own lease and owner audit must pass before `test-run`; without an exact contaminated-sandbox identity or enough distinct clean capacity, the shard remains infrastructure-failed. The test run is `scripts/run-test-shard.mjs` ([sharded-execution.md](./sharded-execution.md)) for suite and selection runs, or a single targeted command for a `target` run. A target executes its registry-declared `preCommand` and then its registry `command` when one is declared, otherwise its native runner, inside one surface process scope carrying the selected surface ID and owner token. Selectors, retry arguments, and the native JSON reporter are forwarded through a declared command, so wrappers such as the headless-Linux Electron launcher remain active remotely. It always writes a target shard summary after `test-run`, including native results and any normalized process leaks, so a native test failure is classified as `failed` rather than a setup failure. The worker exports `CI=1` and `TEST_RETRIES`, so child runners see the `FLAKY_TEST_RETRIES` default ([flaky-tests.md](./flaky-tests.md)). +Each shard runs a script on its worker whose steps are recorded as setup timings: `temporary-cleanup` (the pre-run cleanup above), `checkout` (clone if the worker has no checkout, point `origin` at the run's repository, then fetch and hard-reset to the commit), `runtime-activation`, `runtime-versions`, `deps`, `system-deps` (`xauth`, `libgtk-3-0`, and `iproute2` for `ss`), `owner-lifecycle`, `playwright-cache`, `electron-repair`, then `test-run`. The `electron-repair` phase invokes the installed Electron package's own `install.js` with `/cache/electron` as its archive cache. Electron's installer reuses a complete runtime or prepares an absent one; the phase carries no separate download/extraction implementation and does not delete runtime files. ^blaxel-electron-repair After checkout, `runtime-activation` loads the provisioned nvm and runs `nvm use` from the checked-out `.nvmrc`; it does not run a second version checker or install/repair a runtime. All dependency, repair, and test commands inherit the activated Node/npm path. The `system-deps` step asserts these tools on every lease and installs the package set when any prerequisite is absent. `owner-lifecycle` leaves live-supervisor tokens untouched, reaps stale token-bearing groups with revalidated TERM/KILL, and fails before `test-run` whenever it found contamination. That sandbox is excluded from every later attempt by the coordinator. A configured infrastructure retry may run the shard on a different sandbox, whose own lease and owner audit must pass before `test-run`; without an exact contaminated-sandbox identity or enough distinct clean capacity, the shard remains infrastructure-failed. The test run is `scripts/run-test-shard.mjs` ([sharded-execution.md](./sharded-execution.md)) for suite and selection runs, or a single targeted command for a `target` run. A target executes its registry-declared `preCommand` and then its registry `command` when one is declared, otherwise its native runner, inside one surface process scope carrying the selected surface ID and owner token. Selectors, retry arguments, and the native JSON reporter are forwarded through a declared command, so wrappers such as the headless-Linux Electron launcher remain active remotely. It always writes a target shard summary after `test-run`, including native results and any normalized process leaks, so a native test failure is classified as `failed` rather than a setup failure. The worker exports `CI=1` and `TEST_RETRIES`, so child runners see the `FLAKY_TEST_RETRIES` default ([flaky-tests.md](./flaky-tests.md)). If a setup step fails before the test run and no shard summary was produced, the worker synthesizes an `infra-failed` shard summary naming the failed step (`checkout` failures are reported distinctly). Native result paths rooted at the worker checkout (`/workspace/television`) are normalized to repository-relative paths after download. @@ -138,7 +138,7 @@ Worker execution timeout profiles: `normal` = 240 s; `cold` = 480 s; `repair` = ## Output -The run output directory (default `.blaxel-testshards/latest`) contains `request.json`, combined `report.json`, compact `summary.json`, and any retained per-shard `shard-.log` and `shard-.json` files ([sharded-execution.md](./sharded-execution.md)). A path is exposed only after its local file is retained; shards that never produced a log or result do not advertise an inferred filename. Planned suite runs also contain `shard-plan.json`. Completed shard downloads add `setup-shard-.json` and, when the worker produced it, top-level `native-shard-.json`. Per-surface native reports, Vitest attempt sidecars, lifecycle results, process-leak results, and generated runner inputs are downloaded under `artifacts/shard-/`, preserving their worker-relative paths. Infrastructure retries write `attempt-/` subdirectories with the same applicable layout. Each shard summary exposes normalized restore/wake, checkout, dependency cache hit/miss, system setup, readiness, test-compute, and report-download phases; the coordinator prints those rows and passes them into the canonical run events and summary. Ordinary completion exits `0` only for a passed combined report and `1` for a failed report. A handled `SIGINT` or `SIGTERM` exits `130`; when an attempt exists it writes the interrupted provider report first. When the canonical runner receives that coordinator exit, it finalizes the normalized incomplete run and propagates `130`. +The run output directory (default `.blaxel-testshards/latest`) contains `request.json`, combined `report.json`, compact `summary.json`, and any retained per-shard `shard-.log` and `shard-.json` files ([sharded-execution.md](./sharded-execution.md)). A path is exposed only after its local file is retained; shards that never produced a log or result do not advertise an inferred filename. Planned suite runs also contain `shard-plan.json`. Completed shard downloads add `setup-shard-.json` and, when the worker produced it, top-level `native-shard-.json`. Per-surface native reports, Vitest attempt sidecars, lifecycle results, process-leak results, resource sidecars, recovery receipts, and generated runner inputs are downloaded under `artifacts/shard-/`, preserving their worker-relative paths. Infrastructure retries write `attempt-/` subdirectories with the same applicable layout. Each shard summary exposes normalized restore/wake, checkout, dependency cache hit/miss, system setup, readiness, test-compute, and report-download phases; the coordinator prints those rows and passes them into the canonical run events and summary. Ordinary completion exits `0` only for a passed combined report and `1` for a failed report. A handled `SIGINT` or `SIGTERM` exits `130`; when an attempt exists it writes the interrupted provider report first. When the canonical runner receives that coordinator exit, it finalizes the normalized incomplete run and propagates `130`. ## Operations @@ -155,7 +155,7 @@ The GitHub token the workers use is obtained from the corporate 1Password item * ### Pool provisioning -The canonical runner uses pool `poc`. Provision it by deleting the complete pool with `npm test -- pool blaxel delete --pool poc`, then recreating all 72 fresh workers with `npm test -- pool blaxel ensure --pool poc` ([#Pool](#Pool)). Use `list --pool poc` only for inspection. `ensure` refuses a populated pool and intentionally does not reconcile existing runtime state. A bare command targets the manager's `default` pool, which the canonical runner does not use. +The canonical runner uses pool `poc`. Provision it by deleting the complete pool with `npm test -- pool blaxel delete --pool poc`, then recreating all 72 fresh workers with `npm test -- pool blaxel ensure --pool poc` ([#Pool](#Pool)). Recreation needs an idle pool: deletion reports workers it refused and exits nonzero if any requested worker was not deleted. Use `list --pool poc` for inspection. `ensure` refuses a populated pool and intentionally does not reconcile existing runtime state. A bare command targets the manager's `default` pool, which the canonical runner does not use. ### Opt-in local suite exclusions @@ -169,6 +169,6 @@ Under the testing policy's [rule for integration against paid services or comple A change to pool provisioning or runtime selection requires the live run to use freshly provisioned workers, since `ensure` refuses a populated pool ([#Pool](#Pool)). A dedicated pool may supply this evidence without deleting shared `poc` workers. -Recovery and cleanup experiments use disposable owned processes on a dedicated pool, never intentionally contaminating shared workers. Existing `isolated-github-only` fixtures retain their [placement restriction](./test-runner.md#Destructive fixture placement). Temporary-file placement needs live observation of Node-created paths and Chromium shared-memory descriptor paths under the attempt's `TMPDIR`, with the actual launched browser version and executable recorded; the Playwright package version alone does not establish that browser identity. +Recovery and cleanup experiments use disposable owned processes on a dedicated pool, never intentionally contaminating shared workers. Temporary-file placement needs live observation of Node-created paths and Chromium shared-memory descriptor paths under the attempt's `TMPDIR`. A change to repository selection, or to how a worker's checkout moves between repositories, requires live runs from checkouts of two repositories on the same pool, including a run on workers whose checkout last served the other repository. diff --git a/specs/arch/test-runner/sharded-execution.md b/specs/arch/test-runner/sharded-execution.md index 8657c4ec..825df010 100644 --- a/specs/arch/test-runner/sharded-execution.md +++ b/specs/arch/test-runner/sharded-execution.md @@ -225,7 +225,7 @@ Blaxel workers and regular GitHub CI matrix jobs run `scripts/run-test-shard.mjs - `--fail-fast` stops the shard after the first failed group or task. Every later task that had files assigned by the plan remains in the shard summary with `status: "skipped"`, `skipReason: "fail-fast"`, its assigned files, and no collected files or runner artifacts. - The worker writes `shard-.json` into its results directory, default `.testshards/results`, and exits non-zero if any task failed or plan validation failed. -On Blaxel, the shared worker preserves the outer attempt's ownership and temporary-directory environment; [the Blaxel lifecycle](./blaxel-testshards.md#Locking) owns these, including teardown and download before release. Resource sidecars travel with the task artifacts under [reporting's resource-diagnostics contract](./reporting.md#Resource diagnostics), including evidence from superseded infrastructure attempts. GitHub workers retain their existing host lifecycle. +On Blaxel, the shared worker preserves the outer shard attempt's ownership and temporary-directory environment; [the Blaxel lifecycle](./blaxel-testshards.md#Locking) owns these, including teardown and download before release. Resource sidecars travel with the task artifacts under [reporting's resource-diagnostics contract](./reporting.md#Resource diagnostics), including evidence from superseded infrastructure attempts. GitHub workers retain their existing host lifecycle. ## Blaxel provider report shape diff --git a/specs/arch/test-runner/test-runner.md b/specs/arch/test-runner/test-runner.md index 3a6e584a..17149bfc 100644 --- a/specs/arch/test-runner/test-runner.md +++ b/specs/arch/test-runner/test-runner.md @@ -219,7 +219,7 @@ A combined Vitest run uses a generated config with one `test.projects` entry per Persistent workers inspect Television owner tokens before accepting a new scope. A token is stale only when its recorded supervisor PID and process start time no longer identify a live supervisor; a concurrent run with a live supervisor is left untouched. A stale process is captured and reaped, and the worker reports an infrastructure preflight failure naming the token and process; it does not run a new surface in contaminated state. An ephemeral provider still performs the check so its correctness does not depend on host disposal. -Blaxel additionally reserves the whole worker through result retrieval. Its [abandoned-run recovery](./blaxel-testshards.md#Abandoned-run recovery) may stop that abandoned attempt's live supervisors after coordinator renewal expires; it does not weaken the ordinary stale-owner rule for other runs. Surface supervisors and helpers preserve the outer attempt attribution alongside their own surface tokens. [Reporting](./reporting.md#Resource diagnostics) owns the resource evidence captured across Blaxel surface preparation, execution, and teardown. +Blaxel additionally reserves the whole worker through result retrieval. Its [abandoned-run recovery](./blaxel-testshards.md#Abandoned-run recovery) may stop that abandoned attempt's live supervisors after coordinator renewal expires; it does not weaken the ordinary stale-owner rule for other runs. Surface supervisors and helpers preserve the outer shard attempt attribution alongside their own surface tokens. [Reporting](./reporting.md#Resource diagnostics) owns the resource evidence captured across Blaxel surface preparation, execution, and teardown. ## End-of-surface leak detection diff --git a/specs/arch/testing-policy.md b/specs/arch/testing-policy.md index 8319400f..e291467d 100644 --- a/specs/arch/testing-policy.md +++ b/specs/arch/testing-policy.md @@ -207,7 +207,7 @@ In order to liberally cover boundary seams across the codebase, complex harness Prefer real, committed fixtures over invented data. Better yet, generate fixtures dynamically from current code paths, driven by simpler inputs with fewer assumptions, whenever possible. -Reusable temporary builds belong to the test worker that creates them. The source-CLI runtime is built once per worker and versioned web bundles once per version per worker; test-case cleanup stops that case's servers without discarding these shared builds. Worker teardown waits for pending builds and stops their consumers before removing the outputs, in both Playwright and Vitest. Failed preparation removes its partial runtime and leaves the cache retryable; failed launch preparation also removes its per-launch copy and home. A cached path is published only after preparation succeeds. Cleanup is idempotent and never removes files beneath a surviving consumer. The [Blaxel attempt lifecycle](test-runner/blaxel-testshards.md#Temporary files) supplies the backstop for a killed worker that cannot run its teardown. ^worker-build-lifetime +A reusable build created by a test helper belongs to its worker: it is shared within that worker, published only after preparation succeeds, and removed at worker teardown rather than per test case. The [Blaxel shard attempt lifecycle](test-runner/blaxel-testshards.md#Temporary files) supplies the backstop when a killed worker cannot run its teardown. ^worker-build-lifetime For new or substantially changed browser acceptances that use the `ProductServer` helper, the default is its built application unless the assertion needs development-module behavior. This exercises shipped assets and avoids loading hundreds of development modules for a product assertion. This default does not require retroactive conversion of other tests or apply to in-process `Server` walks using `appURLForServer()`, component fixtures, or Electron module/dispatcher seams. ^product-acceptance-built-app From 3fa15d474b107f97271e5444287037318ce5ee6b Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:59:11 +0000 Subject: [PATCH 07/64] Define live harness test placement and address proof review --- ...arness-resource-cleanup-proof-review-r1.md | 73 +++++++++++++++++++ .../test-harness-resource-cleanup-proposal.md | 4 +- ...arness-resource-cleanup-review-response.md | 25 ++++++- proofs/arch/artifact-frame/artifact-bridge.md | 4 +- proofs/arch/test-runner/blaxel-testshards.md | 4 +- proofs/arch/test-runner/reporting.md | 2 +- proofs/arch/testing-policy.md | 6 +- proofs/arch/themes/delivery.md | 6 +- proofs/index.md | 2 +- proofs/product/artifacts.md | 4 +- proofs/product/themes-and-appearance.md | 12 +-- specs/arch/test-runner/blaxel-testshards.md | 2 +- specs/arch/test-runner/test-registry.md | 5 +- specs/arch/test-runner/test-runner.md | 6 ++ specs/index.md | 4 +- 15 files changed, 130 insertions(+), 29 deletions(-) create mode 100644 docs/working/test-harness-resource-cleanup-proof-review-r1.md diff --git a/docs/working/test-harness-resource-cleanup-proof-review-r1.md b/docs/working/test-harness-resource-cleanup-proof-review-r1.md new file mode 100644 index 00000000..18d973e3 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-proof-review-r1.md @@ -0,0 +1,73 @@ +# Proof review, round 1: test-harness resource cleanup (PR #12, 9fd51f49) + +**Verdict: FAIL, on one item.** The proofs cover every promise the touched specs make, at honest shapes, with every mock named and its forfeit placed, and at a cost in proportion to the risk. The spec refinements since `cd8aa453` change wording and resolve the two inconsistencies my spec review named; they change no converged promise. The one blocker: the live assertions and the deliberately failing fixture tests they run have no home. Which suite holds a test is spec authority (the registry's `kind` list and the runner's suite and exclusion rules), and the proofs decide it silently with "an explicit paid-service selection". That is the one thing a proof may not do, and the tests cannot be written until it is settled. The fix is a few sentences in two specs and one line in the proof, or a different framing that needs no spec change at all. Five refinements follow, none blocking. + +- Reviewed: `git diff cd8aa453 9fd51f49 -- specs proofs docs/working`, the full Blaxel, reporting and testing-policy proofs at `9fd51f49`, the proof policy, the testing policy's mocking, composition, shape and cost rules, the reviewer checklist, and the code the hooks name. +- Branch state: contains `origin/main` (`a8e73fbf`); the copied spec review is identical to mine; product and theme specs are unchanged. +- I ran nothing. Every check is a read of files in the checkout or of git history. + +## The spec refinements change no promise + +| Change since `cd8aa453` | Effect | +| --- | --- | +| *reservation* → *lease*/*lock*; *shard attempt* where one shard is meant; `provider attempt` for the directory identity | Wording only. | +| One rule for unreadable or incomplete lock records: reclaim by acquisition or deletion only after a clean worker audit (`blaxel-testshards.md:57`) | Resolves the contradiction my spec review found; the proposal now says the same (`proposal.md:99`). | +| A never-published fresh worker may be deleted without a lease; a worker that cannot answer the ownership check is held (`:63`) | The narrow half of refinement 3, deliberately; the response explains why it declines the wider inference. Consistent. | +| Legacy paragraph marked transitional; two non-requirements dropped; `temporary-cleanup` named as a setup step; sidecars and receipts located under `artifacts/shard-/`; idle-pool deletion stated in operations; setup sidecars finalized before scratch removal; two `Testing` items removed | All from the spec review; none adds or removes a promise. | +| `^worker-build-lifetime` trimmed to the rule | The dropped sentences were derivable detail; see refinement 4 for one proof assertion left pointing at nothing. | + +## Coverage, shape by shape + +**Blaxel spec.** The lifecycle is composed the way the testing policy asks: breadth at two contracts with authored records and recording callbacks (`^blaxel-t-lease-decisions`, `^blaxel-t-lease-order`), the real handoffs crossed once each with real processes, `flock`, procfs and signals (`^blaxel-t-worker-guard`) and a real filesystem (`^blaxel-t-scratch-cleanup`, with the lease and audit precondition declared as the mock), and the spine walked for real through the canonical CLI against Blaxel (`^blaxel-t-retrieval`, `^blaxel-t-recovery-live`, `^blaxel-t-pool-live`), with the temporary-directory directive honored by inspecting the launched browser's descriptors (`^blaxel-t-tmpdir-live`). I mapped every promise in Locking, Abandoned-run recovery, Older runners and pool operations, Temporary files, Pool, Output and Operations to an assertion; the only two without one are small and noted in refinement 5. The forfeits are honest and placed: mocked clock and audit to the guard seam and live recovery; the audit's token filter to live recovery; SDK substitutions to the pool acceptance. The superseded cleanup bullet (`proofs/arch/test-runner/blaxel-testshards.md:67`) correctly retires the TTL assertion that the existing `test/repo/test-runner-provider.test.ts` still encodes; that test changes with the code. + +**Reporting spec.** Bounds, units, unknown-not-zero, counter resets, retention for failed and recovered attempts, attempt history, link validity, credential exclusion and the receipt's contents are all ordered, at the right price: a writer contract with authored readings, one real parent/child/procfs seam that manufactures no OOM, a normalization contract over real sidecar files, and retrieval shared with the provider's live runs rather than a second crash. The hooks (probe reader, clock, output root, write operation, the guarded registry hook for fixture selection) exist or are declared. + +**Testing policy.** `^worker-build-lifetime` gets a filesystem contract with recording builders plus a real Playwright and Vitest worker seam that builds and serves real output, which is the honest crossing the rule needs. `^product-acceptance-built-app` is honored by the design of the five theme and demo-mode walks and ordered no compliance scan, which is right under `^testing-guidance-standing`. + +**Shared execution, runner, themes, artifacts.** Cross-references only, pointing at the provider and reporting assertions; the five converted walks keep their product assertions and change only their loading boundary, which the proof policy allows a proof to do alone (`^proof-change`). All new anchors are unique, every link resolves, cited test files exist, the `TV_TEST_RUNNER_SELFTEST` gate matches its existing use (`scripts/test/cli.mjs:1074`, `planned-shard-worker.mjs:458`), and the proofs index matches the files. + +## Blocker: the live assertions and their fixtures have no home + +`^blaxel-t-retrieval`, `^blaxel-t-recovery-live`, `^blaxel-t-pool-live`, `^blaxel-t-tmpdir-live` and `^reporting-t-resource-live` are automated tests: they spawn the canonical CLI, provision a pool, kill a coordinator, and "a bounded test driver owns and finally cleans up its coordinators and private workers". They need Blaxel credentials, a dedicated pool and minutes of wall clock, and they run "committed native fixture tests" that fail once or always. The coverage model says only that they are "an explicit paid-service selection, not a recursive addition to every broad run" (`proofs/arch/test-runner/blaxel-testshards.md:15`). + +- Which suite holds a surface is spec-owned: `test-registry.md:44` enumerates the kinds in use, `:95` says what `all` and the `experiment` suite contain and why, `test-runner.md:61, 92` name the opt-in suites, and `blaxel-testshards.md:162` lists the suites excluded from Blaxel broad verification. None of them admits a surface that drives Blaxel from inside a test, or fixture surfaces that fail by design. +- The fixtures have the same gap. A planned Blaxel run selects surfaces from the registry on the worker; a fail-once fixture inside `all` would break every broad run, and the reporting proof's registry hook (`TEST_SHARD_CONFIG_MODULE`) reaches a local parent, not a Blaxel worker, unless the coordinator forwards it, which would be a new hook the Blaxel proof does not declare. +- Where they run also matters here: the assignment and Josh's rule keep tests off the shared developer host, and the proposal names a credentialed Blaxel coordinator sandbox. The proof should say that these tests are run from such a host, never from a shared developer host or a Blaxel worker. + +Two ways to clear it, either small: + +1. **Register them.** Add an opt-in surface for the live harness tests and one for the fixture tests (an existing kind such as `experiment`, whose spec already carries one "runs only when selected" class, or a new kind), with the registry, runner and Blaxel specs updated to say so, and the proof's shared live declaration naming the surfaces and the host they run from. Josh sees the spec sentences at PR time with the rest. +2. **Or make them evidence, not tests.** The Blaxel spec's `Testing` section already requires operator-run live evidence for coordinator changes and says what it must exercise. The live scenarios could be ordered the same way: a recorded run from the credentialed coordinator host, with the receipts and run directories as the evidence, and only the fixture surfaces registered. That removes the test driver and its cleanup obligations at the cost of repeatability. + +The proofs should raise this as a finding against the spec rather than choose; I am raising it for them. Everything else in the live assertions stands as written. + +## Refinements (none blocking) + +1. **Grade the five converted walks honestly at the assertion.** `^theme-ac-selection`, `^theme-ac-active-package`, `^theme-delivery-t-active-package-refresh` and the demo-mode shared declaration now describe the built application and cite the existing tests as "covered by", while the coverage models say the adaptation is "(test to be written)". A reader of the assertion alone sees policy-grade coverage of a boundary no test crosses yet. The pool-pagination assertion shows the right form: "(policy-grade test for the existing walk: …; built-application boundary: test to be written)". The theme lines should also name the stable-front proxy, which the coverage model declares but the assertions' "no mocks or test hooks" omits; it is harness transport, not a replaced mechanism, and belongs in the declaration. + +2. **`^blaxel-t-worker-guard` should say where it runs.** "In a private fixture root on a dedicated worker" reads as a live requirement. Everything the seam uses, `flock`, procfs, signals, a filesystem, exists on any Linux CI host, so it should be an ordinary repository test on Linux, like the stale-owner audit's live cases. Say so; otherwise the cheapest real crossing in the proof is priced like an acceptance. + +3. **State the worker-side fixture selection.** Once the blocker settles the surfaces, `^blaxel-t-retrieval` should say the planned run selects them with `--suite selection --surfaces …`, which needs no new hook, so an implementer does not reach for forwarding the registry hook to workers. + +4. **`^testing-t-launch-cleanup` points at nothing.** It cites the policy's "Timing, observable status, cleanup, and flakiness" section, which is two sentences about readiness and sleeps, and the spec sentence it used to derive from was trimmed at the spec review. Under the proof policy that makes it a promise living only in the proof: either fold the per-launch copy into `^testing-t-build-lifetime` as one more owned output under `^worker-build-lifetime`, or drop it. `^testing-t-build-lifetime`'s second pointer to the same section is harmless but adds nothing. + +5. **Two small promises without an assertion.** "Uploads use paths specific to the shard attempt" (`blaxel-testshards.md:43`) and the `temporary-cleanup` setup timing (`:86`) can each be one observation inside `^blaxel-t-retrieval`: the uploaded script's path lies inside the attempt directory, and the shard summary's phases include the cleanup step with its counts. Both cost nothing extra on a run that already happens. + +## Reviewer checklist + +- **Proof coverage.** Every promise at an honest shape; directives honored and the coverage models say where; mocks declared with forfeits; coverage carried elsewhere named. The two gaps are refinement 5. +- **Mocking and boundaries.** No mock sits inside the chain an assertion is about. The audit-filter mock in the guard seam and the lease/audit precondition in the cleanup seam are declared and forfeited to the live runs. Real boundaries (process, filesystem, Blaxel API, browser) are each crossed for real at least once. +- **Placement.** Nothing spec-shaped hides in the proofs except the suite decision in the blocker. +- **Status and citation grades.** New obligations carry "(test to be written)"; the five converted walks need the mixed form (refinement 1). +- **References.** All resolve; the proofs index is regenerated and correct. +- **Cost.** Four live scenarios on a two-worker pool, sharing fixtures; one real nested-runner seam that rebuilds the source CLI once; the rest contracts and cheap seams. Proportionate to a change whose whole risk is reuse before owned processes stop or evidence reaches the caller. + +## Evidence and limits + +- Read every proof hunk with context and the three rewritten proofs in full; checked each new assertion against the spec paragraph it cites and each spec promise against the assertions; verified anchors, links, cited files, the self-test gate and the index by `git grep` and `git show`. +- Not established: whether Blaxel accepts a label change on a `DEPLOYED` sandbox, which `^blaxel-t-pool-live` will show; the exact `experiment` placement rules beyond what `test-registry.md:95` states. + +## Housekeeping + +- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel. `git fetch origin` updated remote-tracking refs. +- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-proposal.md b/docs/working/test-harness-resource-cleanup-proposal.md index fde4e64e..6df5420a 100644 --- a/docs/working/test-harness-resource-cleanup-proposal.md +++ b/docs/working/test-harness-resource-cleanup-proposal.md @@ -211,7 +211,7 @@ After proposal convergence, derive the smallest changes in the existing owners: - [`blaxel-testshards.md`](../../specs/arch/test-runner/blaxel-testshards.md): exclusive reservation through setup, process teardown and result retrieval; renewal, automatic recovery after lost renewal, owned-process cleanup before reuse, and the explicit abandoned-report exception; actual older-runner behavior; pool-script locking; attempt temporary directories, recurring exact-prefix cleanup and bounded worker retention. Replace its blind TTL escape and shell-exit unlock expectations rather than leaving conflicting requirements. - [`reporting.md`](../../specs/arch/test-runner/reporting.md): minimum resource observations and artifact references, bounded content, unavailable evidence, retention alongside failed/recovered native attempts, and recovery receipts that disclose discarded orphan reports. -Use existing lifecycle and reporting contracts in `test-runner.md` and `sharded-execution.md`, adding only the cross-references or ownership corrections required by the outer Blaxel lock. Existing isolated-GitHub-only lifecycle fixtures keep their placement. No `preflight.md` delta is planned. Optional investigation instrumentation remains optional. +Use existing lifecycle and reporting contracts in `test-runner.md` and `sharded-execution.md`, adding the cross-references or ownership corrections required by the outer Blaxel lock. The proof review also exposed missing authority for the automated live driver and its deliberate-failure fixtures: the proposed `test-registry.md` and `test-runner.md` delta places them in separately selected experiment surfaces, with the driver on a dedicated credentialed Blaxel coordinator and the fixtures on its private pool. This keeps repeatable live coverage without adding a kind or suite. That placement delta awaits independent spec review before dependent proof derivation. Existing isolated-GitHub-only lifecycle fixtures keep their placement. No `preflight.md` delta is planned. Optional investigation instrumentation remains optional. Update the mirrored proofs after independent spec convergence. Test setup changes may also require proof-only corrections to descriptions of the affected browser walks; they do not change their product promises. Do not prescribe a new framework or adopt unrelated harness internals into spec authority. @@ -249,4 +249,4 @@ Proposal round 2 at `d3570bf4`: FAIL because dead-coordinator lock recovery is u Proposal round 3 at `57a701d0`: **PASS**, relayed by the supervisor. The [review](test-harness-resource-cleanup-review-r3.md) is preserved unchanged. Its bounded refinements are applied here: fixed recovery interval, deletion of abandoned workers despite failed process cleanup, ordinary skipping of live legacy work, and trimmed receipt/provisioning language. The [response](test-harness-resource-cleanup-review-response.md#Round 3: convergence and spec derivation) records the judgments. The proposal and spec deltas have converged: spec round 1 at `cd8aa453` also passed. Its bounded refinements and the derived proof checkpoint are recorded in the response; independent proof review is next. -Proof derivation is complete for review; test and implementation work has not begun. No tests, builds, remote experiments, or worker operations were performed for this checkpoint. Unknowns are live provider behavior during failed dispatch/recovery, today's worker browser versions, actual `TMPDIR` placement and post-mitigation memory savings, current capacity/credential availability, and observed recovery timing. Recovery waits at most 20 minutes after the last renewal by default, then acquisition and bounded cleanup; failed cleanup prevents test reuse and explicit pool deletion remains available. Josh's spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. +Proof round 1 at `9fd51f49`: **FAIL** on missing spec-owned placement for live qualification and deliberate-failure fixtures. The [review](test-harness-resource-cleanup-proof-review-r1.md) is preserved unchanged. The [response](test-harness-resource-cleanup-review-response.md#Proof round 1: placement finding and refinements) proposes the narrow placement delta for independent spec review and applies the independent proof refinements. Dependent live-proof binding follows that review; test and implementation work has not begun. No tests, builds, remote experiments, or worker operations were performed for this checkpoint. Unknowns are live provider behavior during failed dispatch/recovery, today's worker browser versions, actual `TMPDIR` placement and post-mitigation memory savings, current capacity/credential availability, and observed recovery timing. Recovery waits at most 20 minutes after the last renewal by default, then acquisition and bounded cleanup; failed cleanup prevents test reuse and explicit pool deletion remains available. Josh's spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index fcdb8396..8b249ef3 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -1,6 +1,6 @@ # Resource-cleanup proposal: review response -Proposal round 3 at `57a701d0` and spec round 1 at `cd8aa453` both **PASS**, relayed by the supervisor. Proposal and spec deltas have converged; the proof checkpoint below is ready for independent review. The round 1 and 2 sections below retain the response history; [round 3](#Round 3: convergence and spec derivation) records the bounded refinements and current spec checkpoint. Reviewer: Claude Fable (Nina), arranged by the supervisor. +Proposal round 3 at `57a701d0` and spec round 1 at `cd8aa453` both **PASS**, relayed by the supervisor. Proof round 1 at `9fd51f49` **FAILS** on missing spec-owned placement for the live tests and their deliberate-failure fixtures. The [proof response below](#Proof round 1: placement finding and refinements) returns that narrow delta to spec review and records the independent proof corrections. Earlier sections retain the response history. Reviewer: Claude Fable (Nina), arranged by the supervisor. Independent round 1 reviewed `1652787e`. The [full review](test-harness-resource-cleanup-review-r1.md) remains unchanged: FAIL on two blockers. The response committed at `99b7ed93` narrowed the work pending Josh's decisions. The following sections through round 2 record that history, not the current review verdict. @@ -130,3 +130,26 @@ The main risk worth the live cost is reuse before owned processes stop or eviden Validation for this checkpoint is document/source inspection, review-copy comparison, Markdown index regeneration, and Git whitespace/diff inspection. No tests, builds, browser/server launches, provider experiments, or worker operations ran. Test and harness implementation has not begun. Independent proof review is the next gate; Josh's spec review remains at PR time. Primitive's file check surfaced existing recovery decisions and the correction to the supervisor's illustrative rule, with `truncated: true` from discovery. That was not a complete constraint check; no decision was overridden or reconciled. These proof details derive the existing choices and introduce no new higher-order direction to record. + +## Proof round 1: placement finding and refinements + +The [complete proof review](test-harness-resource-cleanup-proof-review-r1.md) is copied verbatim. Its one blocker is warranted: the proofs ordered automated live qualification and intentionally failing native fixtures without an owning registry/runner placement rule. The worker reads the committed registry; the local replacement-registry hook is not a worker-selection mechanism. + +The proposed correction keeps automated qualification and uses the existing `experiment` kind. Register separate `experiment:blaxel-harness` and `experiment:blaxel-harness-fixtures` surfaces; both stay outside `all` and ordinary CI. The driver runs through the canonical local runner on a dedicated credentialed Blaxel coordinator outside the worker pool and owns a disposable private pool. Its worker selection contains only the native fixtures. This retains repeatability, unlike operator-only live evidence, without a new kind, suite, forwarded registry hook, or general execution-placement framework. Ordinary Linux seams remain in the repository suite. + +The delta is in `test-registry.md` (surface ownership/exclusion), `test-runner.md` (coordinator placement, planned/targeted fixture selection and driver cleanup), and a reference from `blaxel-testshards.md`. It changes no lock, recovery, cleanup or diagnostic promise. It does need independent spec review: the earlier PASS did not cover this missing placement. [tvdev-contribute](/home/user/.agents/skills/tvdev-contribute/SKILL.md) requires “independent agent convergence of every spec delta remains required before proof derivation.” The corresponding repository [workflow](../../specs/spec-workflow.md#Recommended approach) governs that ordering. Josh's human spec review remains at PR time. + +| Finding | Assessment and disposition | +| --- | --- | +| B1: live tests and deliberately failing fixtures have no home | Return the proposed placement above to independent spec review before binding the live proof declarations to it. The existing behavioral live assertions remain pending obligations; no tests or driver implementation are written at this checkpoint. | +| R1: grades and proxy declaration | Accepted. Every affected theme, delivery, artifact and bridge assertion now distinguishes its existing policy-grade walk from the unwritten built-application adaptation. The theme declarations name the real stable-front proxy as harness transport; the coverage model still explains stable origin/authentication across port-0 backend restarts. Reset and already-built walks retain their coverage grades. | +| R2: worker-guard seam placement | Accepted. State ordinary repository test on Linux, with real procfs, signals, OS guard and filesystem. No provider credentials or live-provider acceptance is required. This contribution still executes that test only on Blaxel. The whole-worker audit forfeit remains with live recovery. | +| R3: worker fixture selection | Accepted in the proposed owning specs. A canonical `--tag blaxel-harness-fixture` selection delegates `--suite selection --surfaces experiment:blaxel-harness-fixtures`; `--surface experiment:blaxel-harness-fixtures` selects the targeted path. Bind those commands into the shared live declaration and retrieval assertion after placement convergence. Clarify now that reporting's replacement-registry hook applies only to the local parent/child seam. | +| R4: failed-launch cleanup assertion | Consolidate it into the existing real worker seam, using the missing bundle path and subsequent valid launch within the same fixture; remove the standalone assertion and redundant second contract citation. The review's authority premise is too narrow: the testing policy's timing/cleanup section does say cleanup of long-running modules is core to correctness, below its motion subsection. The consolidation is still worthwhile because the same worker-lifetime crossing can cover the private-copy failure without another fixture or launch sequence. | +| R5: upload path and cleanup timing | Accepted. Add to the existing live retrieval run: uploaded script paths are inside the current shard attempt directory, and retained phases include `temporary-cleanup` with counts and free-space evidence. No new live run. | + +After placement spec convergence, complete the dependent proof binding: name both surfaces and the coordinator host in the shared live declaration, spell out planned/targeted selection in retrieval, and derive selection coverage for exclusion from `all`/ordinary CI and absence of driver recursion. Reporting's live diagnostic assertion shares that same driver. These are pending derivation steps, not silently approved additions to the reviewed proof tree. + +Validation is document/source inspection, exact comparison of the saved review, Markdown index regeneration and Git diff/whitespace inspection. No tests, builds, browser/server launches, provider experiments or worker operations ran. Test and harness implementation remains unstarted. + +Primitive's file check returned existing recovery and placement decisions with `truncated: true`; it was not a complete constraint check. No decision was overridden or reconciled. The latest 20 decisions contained no duplicate of the proposed automated experiment placement; it is recorded as agent-attributed `dec_7893f958`, with the repeatability rationale and operator-only alternative. That record does not substitute for independent spec convergence or Josh's PR review. diff --git a/proofs/arch/artifact-frame/artifact-bridge.md b/proofs/arch/artifact-frame/artifact-bridge.md index 2f1af862..1d9ad2d5 100644 --- a/proofs/arch/artifact-frame/artifact-bridge.md +++ b/proofs/arch/artifact-frame/artifact-bridge.md @@ -79,7 +79,7 @@ Readiness assertions observe lifecycle state, never shielding, pointer suspensio - **Contract** (browser-bridge contract harness; the parent is mocked; authored platform identities cover macOS and Windows/Linux): the bridge captures option+arrow on macOS and ctrl+arrow on Windows/Linux for all four arrows, including in text-editing targets and with `repeat`; calls `preventDefault`; and posts `navigation-key`. Shift, every other extra modifier, composition, and non-arrow keys do not forward, while a descendant handler's `stopPropagation()` cannot starve the window capture listener — *(covered by tests in `packages/artifact/test/unit/artifact-bridge.test.ts`: "matches the platform navigation chord for every arrow", "rejects composition, non-arrows, missing chords, and every extra modifier", "captures matching chords in editing targets without stopping propagation", "leaves unmatched keydown behavior untouched", and "produces runnable source that behaves like installBridge when eval'd in a fresh window")*. ^ab-ac-navigation-key - **Contract** (browser-bridge and preload contract harnesses; parent `postMessage` and Electron IPC peers are mocked, forfeiting real handoff to the acceptance assertions below): each producer captures trusted pointer move, down, up, cancel, and click events and emits one `artifact-pointer` carrying the raw guest-viewport coordinates and button fields without preventing default, stopping propagation, or taking pointer capture; untrusted events are ignored — *(covered by `packages/artifact/test/unit/artifact-bridge.test.ts` “reports trusted artifact pointer input without interfering” and `packages/desktop/test/webview-bridge-preload.test.ts` “reports trusted webview pointer input without interfering”)*. ^ab-ac-pointer-producers - **Contract** (host contract harness plus production theme frames in a browser DOM; source windows, frame geometry, and pointer notifications are authored inputs, while browsing-context delivery is forfeited to the acceptance assertions): the current iframe or subscribed webview's source-valid `artifact-pointer` is accepted without readiness, mapped from guest coordinates through the frame's rendered rectangle and layout dimensions, normalized by the theme owner, and delivered once to each current theme frame. A zero layout dimension drops it, and the same payload from another window or replaced webview is ignored — *(covered by `packages/web/test/artifact-view.test.ts` “relays only current artifact pointer input to theme frames in application coordinates”)*. ^ab-ac-pointer-host -- **Acceptance** (outer boundary: physical pointer input in a real Chromium artifact iframe against a running production server and real theme package; the artifact fixture records its trusted event coordinates without altering input or delivery, and no mocks or production test hooks are used): move, down, up, and click cross the injected bridge and trusted channel to both theme frames in application-viewport coordinates, mapped from the coordinates actually received by the artifact (including Chromium's integer click coordinates), while the artifact's own click action remains effective — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^ab-ac-pointer-browser +- **Acceptance** (outer boundary: physical pointer input in a real Chromium artifact iframe within the built application, served through the real stable-front proxy as harness transport, against a running production server and real theme package; the artifact fixture records its trusted event coordinates without altering input or delivery, and no mocks or production test hooks are used): move, down, up, and click cross the injected bridge and trusted channel to both theme frames in application-viewport coordinates, mapped from the coordinates actually received by the artifact (including Chromium's integer click coordinates), while the artifact's own click action remains effective — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. ^ab-ac-pointer-browser - **Acceptance** (outer boundary: native mouse input in an ordinary HTML artifact webview in the real Electron app against a running production server and real theme package; no mocks or test hooks): the preload and subscribed IPC path deliver move, down, up, and click to both theme frames in application-viewport coordinates while the artifact's own click action remains effective — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “executes an executable theme in Electron and resets on remote consent and selection changes”)*. ^ab-ac-pointer-electron - **Seam** (browser bridge → host navigation handler, crossed by real `postMessage` in the browser seam harness; an authored document and tab/channel state are fixtures, with no mock on the handoff): each of the four forwarded key values reaches the app's navigation handling while focus is inside the document — *(covered by test: `packages/web/test/e2e/artifact-view.test.ts` "forwards every browser navigation arrow from a focused artifact document")*; product outcomes compose with [product/keyboard-navigation.md](../../../specs/product/keyboard-navigation.md). ^ab-ac-navigation-key-seam - **Contract** (desktop shell side of the native-key delivery seam; authored native key events and platform identities cover macOS and Windows/Linux; the renderer navigation receiver is mocked, forfeiting the real handoff to [#^ab-ac-electron-key-seam](#^ab-ac-electron-key-seam)): the matcher implements the same modifier, arrow, repeat, and composition matrix as [#^ab-ac-navigation-key](#^ab-ac-navigation-key), consumes a match, and delivers its arrow value once to app navigation — *(covered by tests: all cases under `packages/desktop/test/native-navigation-key.test.ts` "Electron native navigation chord", plus `packages/desktop/test/main.test.ts` "consumes a matching webview chord and sends its arrow once to the embedding renderer" and `packages/web/test/renderer.test.ts` "delivers Electron native arrows to the existing application navigation edge")*. ^ab-ac-electron-key-contract @@ -116,6 +116,6 @@ The recorder's two branches hand their reports to the host through one function, Demo-mode frames are not bridge participants ([^ab-demo-frame-inert](../../../specs/arch/artifact-frame/artifact-bridge.md#^ab-demo-frame-inert)): -- **Seam** (Chromium against a really-running server under the server-home hook and cross-origin fixture server declared in [product/artifacts.md](../../product/artifacts.md#test-hooks); two external page artifacts on one channel and a second, newer channel are fixtures created through the production API; code evaluated inside the fixture page's frame posts bridge-shaped `bridge-ready`, a horizontal and a vertical `navigation-key` each chosen to have somewhere to move, `navigation-request`, and `artifact-pointer` messages to its parent through real `postMessage`; the default motion override is the only mock): after the page posts them, the focused channel and selected tab are unchanged, the fixture's first page is still framed at its own URL, and Back and Forward remain absent. Theme pointer delivery is not observed here; [#^ab-ac-demo-inert-host](#^ab-ac-demo-inert-host) carries it — *(covered by test: `packages/web/test/e2e/browser-demo-mode.test.ts` "bridge-shaped messages from a demo-mode page change nothing (^ab-ac-demo-inert-seam)")*. ^ab-ac-demo-inert-seam +- **Seam** (the built application in Chromium against a really-running server under the server-home hook and cross-origin fixture server declared in [product/artifacts.md](../../product/artifacts.md#test-hooks); two external page artifacts on one channel and a second, newer channel are fixtures created through the production API; code evaluated inside the fixture page's frame posts bridge-shaped `bridge-ready`, a horizontal and a vertical `navigation-key` each chosen to have somewhere to move, `navigation-request`, and `artifact-pointer` messages to its parent through real `postMessage`; the default motion override is the only mock): after the page posts them, the focused channel and selected tab are unchanged, the fixture's first page is still framed at its own URL, and Back and Forward remain absent. Theme pointer delivery is not observed here; [#^ab-ac-demo-inert-host](#^ab-ac-demo-inert-host) carries it — *(policy-grade test for the existing walk: `packages/web/test/e2e/browser-demo-mode.test.ts` "bridge-shaped messages from a demo-mode page change nothing (^ab-ac-demo-inert-seam)"; built-application boundary: test to be written)*. ^ab-ac-demo-inert-seam - **Contract** (host contract harness; an authored external page artifact posed under browser demo mode, with synthetic `MessageEvent`s whose source is the frame's own window; the transport mock forfeits the real crossing to [#^ab-ac-demo-inert-seam](#^ab-ac-demo-inert-seam)): every message type in the [message protocol](../../../specs/arch/artifact-frame/artifact-bridge.md#message-protocol) produces no navigation-handler, pointer-forwarder, navigation-state, reload, or status-reply call and leaves the frame's trusted identity null — *(covered by test: `packages/web/test/artifact-view.test.ts` "browser demo mode artifact frame > acts on no bridge message from a demo-mode frame (^ab-ac-demo-inert-host)")*. ^ab-ac-demo-inert-host diff --git a/proofs/arch/test-runner/blaxel-testshards.md b/proofs/arch/test-runner/blaxel-testshards.md index f1e0531a..1a1b1ac2 100644 --- a/proofs/arch/test-runner/blaxel-testshards.md +++ b/proofs/arch/test-runner/blaxel-testshards.md @@ -75,7 +75,7 @@ The remaining coverage lives in `test/repo/test-runner-provider.test.ts`. - **Contract — coordinator ordering and failures.** The coordinator's lease lifecycle uses recording/failing transport, download, and cleanup callbacks. Under [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release), renewal spans acquisition, execution, cleanup, and retrieval. A successful shell exit still requires descendant confirmation; unknown dispatch or failed cleanup cannot release. Successful retrieval and identity validation precede release; a download retry neither reruns tests nor resets worker results. An unused lease waits for pending writes and retains produced evidence. On interruption, the authoritative interrupted report precedes cleanup/retrieval and cannot become an ordinary completion report; cleanup still runs after a report-write error. A coordinator that loses ownership stops issuing mutations and reports infrastructure failure. These callbacks forfeit API, signals, and downloads to the live retrieval/recovery assertions; existing signal-report fixtures supply breadth without another native suite. (test to be written) ^blaxel-t-lease-order -- **Seam — guarded registration and ownership.** Production worker-entry and lock helpers cross real Linux process/filesystem boundaries in a private fixture root on a dedicated worker. Two competing acquisitions yield only one owner; killing a process holding the short guard releases the OS guard without deleting another lease. A request paused before entry registration, then resumed after ownership changes, cannot touch a checkout sentinel. Old-ID renewal/release cannot change the new record. Registered setup descendants preserve the outer token when a nested surface replaces its own token; recovery stops only the attributed PID/start identities, including a descendant that ignores TERM and needs KILL. A separate live fixture owner remains untouched and prevents reuse. The fixtures are small processes, records produced by the real helper, and rendezvous files. The supplied audit filters real procfs observations to the fixture tokens, a mock of whole-worker scope that forfeits that protection to live recovery; process inspection, identity revalidation, signals and filesystem operations remain real. This proves [registration](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-reservation) and [recovery](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery), not provider dispatch. (test to be written) ^blaxel-t-worker-guard +- **Seam — guarded registration and ownership.** Production worker-entry and lock helpers cross real Linux process/filesystem boundaries in a private fixture root as an ordinary repository test on Linux. It requires no Blaxel API or credentials; it runs only on Linux because it uses procfs, signals and the OS guard. Two competing acquisitions yield only one owner; killing a process holding the short guard releases the OS guard without deleting another lease. A request paused before entry registration, then resumed after ownership changes, cannot touch a checkout sentinel. Old-ID renewal/release cannot change the new record. Registered setup descendants preserve the outer token when a nested surface replaces its own token; recovery stops only the attributed PID/start identities, including a descendant that ignores TERM and needs KILL. A separate live fixture owner remains untouched and prevents reuse. The fixtures are small processes, records produced by the real helper, and rendezvous files. The supplied audit filters real procfs observations to the fixture tokens, a mock of whole-worker scope that forfeits that protection to live recovery; process inspection, identity revalidation, signals and filesystem operations remain real. This proves [registration](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-reservation) and [recovery](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery), not provider dispatch. (test to be written) ^blaxel-t-worker-guard - **Seam — bounded filesystem cleanup.** The production cleaner crosses the real filesystem in a disposable parent containing both allowed legacy prefixes, stale scratch with complete and missing metadata, the current scratch, unrelated files/directories, and symlinks to outside sentinels. With a lease and clean-audit precondition supplied as a mock, it removes exactly the permitted real directories, leaves the current directory and outside sentinels intact, rejects symlink roots, and records counts, allocated bytes and free-space observations under [known leftovers](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-known-leftovers). The audit mock forfeits real disuse confirmation to the live retrieval/recovery cases. No assertion equates allocated bytes with the filesystem's free-space increase. (test to be written) ^blaxel-t-scratch-cleanup @@ -83,7 +83,7 @@ The remaining coverage lives in `test/repo/test-runner-provider.test.ts`. Shared live declaration: the following acceptances spawn the canonical CLI with real argv and inspect its exit and retained run directory. A fresh dedicated pool, committed native fixture tests, known leftover directories and unrelated sentinels are fixtures. The coordinator, SDK, worker scripts, native runners, transport, ownership audits and filesystem operations are real. The report-download rendezvous above is the only scheduling hook; no mechanism is mocked. A bounded test driver owns and finally cleans up its coordinators and private workers. These tests do not invoke fixtures restricted to isolated GitHub hosts. -- **Acceptance — reports before reuse.** A small planned run with at least two shards traverses real acquisition, architecture verification, checkout, runtime/dependency setup, execution and log/report transport under [the shard lifecycle](../../../specs/arch/test-runner/blaxel-testshards.md#Shard lifecycle). While it waits at the download rendezvous, observed renewals keep the leases valid; a competing canonical run cannot acquire the occupied capacity or alter its result files. After retrieval, the original native reports, setup/cleanup evidence and diagnostics have the original identities and readable local paths, scratch is gone, and the workers become available to a targeted run. Legacy-prefix and stale-scratch fixtures disappear, unrelated sentinels and persistent caches survive, and the next run does not accumulate an older results tree. The targeted run crosses its distinct dispatch path too. This proves [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release) and [scratch lifetime](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-scratch); the shared diagnostics observations are owned by [reporting](reporting.md#^reporting-t-resource-live). (test to be written) ^blaxel-t-retrieval +- **Acceptance — reports before reuse.** A small planned run with at least two shards traverses real acquisition, architecture verification, checkout, runtime/dependency setup, execution and log/report transport under [the shard lifecycle](../../../specs/arch/test-runner/blaxel-testshards.md#Shard lifecycle). Each uploaded script path lies inside its current shard attempt directory. The retained setup phases include `temporary-cleanup`, with cleanup counts and free-space evidence. While it waits at the download rendezvous, observed renewals keep the leases valid; a competing canonical run cannot acquire the occupied capacity or alter its result files. After retrieval, the original native reports, setup/cleanup evidence and diagnostics have the original identities and readable local paths, scratch is gone, and the workers become available to a targeted run. Legacy-prefix and stale-scratch fixtures disappear, unrelated sentinels and persistent caches survive, and the next run does not accumulate an older results tree. The targeted run crosses its distinct dispatch path too. This proves [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release) and [scratch lifetime](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-scratch); the shared diagnostics observations are owned by [reporting](reporting.md#^reporting-t-resource-live). (test to be written) ^blaxel-t-retrieval - **Acceptance — abandoned run recovery.** In two small scenarios, kill the real coordinator: once while a shard supervisor and a token-bearing child remain, and once after the shard stops but before download. The next canonical run, after the shortened configured recovery interval, stops the old owned processes where present, confirms a clean worker, clears allowed residue, and runs on that worker. Its downloaded recovery receipt retains the old identity, renewal/deadline, cleanup result and discarded-report flag across results reset. No old report becomes a pass. This proves the same [automatic recovery path](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery) works for active and quiet orphans without waiting for their voluntary exit. Failure permutations are carried by the contracts, not more live crash runs. (test to be written) ^blaxel-t-recovery-live diff --git a/proofs/arch/test-runner/reporting.md b/proofs/arch/test-runner/reporting.md index eb3d99b4..53c36714 100644 --- a/proofs/arch/test-runner/reporting.md +++ b/proofs/arch/test-runner/reporting.md @@ -19,7 +19,7 @@ Planned and targeted execution both cross parent capture and download. The same `TV_VITEST_ATTEMPT_FILE` supplies the NDJSON output path to the production Vitest attempt recorder. The canonical runner and the planned shard worker set it to the current native run's attempt-result path. A Vitest invocation that configures the recorder outside those entrypoints must set it; the recorder fails without it. -The resource collector accepts a probe reader and clock; contract tests provide authored readings/errors and times, while production reads Linux counters, memory information and filesystem statistics at the current time. The writer accepts an output root and write operation; tests use disposable directories and may inject a write failure, while production writes into the current results tree. Native fixture selection uses the runner's existing guarded registry hook, not a replacement reporter or fabricated native result. +The resource collector accepts a probe reader and clock; contract tests provide authored readings/errors and times, while production reads Linux counters, memory information and filesystem statistics at the current time. The writer accepts an output root and write operation; tests use disposable directories and may inject a write failure, while production writes into the current results tree. Local parent/child seam fixtures use the runner’s existing guarded registry hook, not a replacement reporter or fabricated native result. That hook is not forwarded to Blaxel workers. ## Assertions diff --git a/proofs/arch/testing-policy.md b/proofs/arch/testing-policy.md index 3896ca72..9043e7bc 100644 --- a/proofs/arch/testing-policy.md +++ b/proofs/arch/testing-policy.md @@ -20,8 +20,6 @@ The shared build-lifetime helper accepts a temporary parent and preparation call ## Worker-owned builds -- **Contract — publish successful builds and dispose owned work.** With a real temporary filesystem and recording preparation callbacks, concurrent requests in one worker reuse a pending/successful build for the same key, while distinct version keys remain distinct. A failed preparation removes its partial output and is not published; a later request may prepare it again. Teardown waits for in-flight preparation to settle and removes all owned outputs, including one that failed after creating files. Per-case server disposal cannot invalidate the worker's reusable build. These callbacks mock build/asset preparation and forfeit actual executable output to the worker seam below. This proves [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime) and [cleanup correctness](../../specs/arch/testing-policy.md#Timing, observable status, cleanup, and flakiness). (test to be written) ^testing-t-build-lifetime +- **Contract — publish successful builds and dispose owned work.** With a real temporary filesystem and recording preparation callbacks, concurrent requests in one worker reuse a pending/successful build for the same key, while distinct version keys remain distinct. A failed preparation removes its partial output and is not published; a later request may prepare it again. Teardown waits for in-flight preparation to settle and removes all owned outputs, including one that failed after creating files. Per-case server disposal cannot invalidate the worker's reusable build. These callbacks mock build/asset preparation and forfeit actual executable output to the worker seam below. This proves [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime). (test to be written) ^testing-t-build-lifetime -- **Seam — real runner workers own their builds.** The installed Playwright and Vitest runners each execute a small authored fixture through the actual shared helper/adaptor path, using a private temporary root. Two cases assigned to the same worker reuse the source-CLI preparation and a same-version web bundle; a later worker owns a separate lifetime. Source CLI output starts a real product server, and the real web output is served/read, establishing that published results are usable. After per-case server cleanup the shared outputs still exist; after normal worker teardown they do not. No runner, builder, filesystem or server mechanism is mocked. This crosses each framework's teardown boundary under [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime), without a browser or an extra product assertion. [The real version-stamp seam](updates/version-advertisement.md#^t-web-stamp) supplies version breadth; this seam need not rebuild a matrix of versions. (test to be written) ^testing-t-build-worker-seam - -- **Seam — launch preparation failure leaves no private copy.** Through the real product-server helper, a supplied bundle path naming an absent child in a temporary fixture directory makes the real copy fail after allocating the per-launch runtime/home. The filesystem contains no surviving per-launch output, and a subsequent valid launch can use the still-live worker build. No filesystem cleanup or launch mechanism is replaced; the missing bundle path is fixture data. This proves helper cleanup under [Timing, observable status, cleanup, and flakiness](../../specs/arch/testing-policy.md#Timing, observable status, cleanup, and flakiness), while the successful real launch belongs to the worker seam above. (test to be written) ^testing-t-launch-cleanup +- **Seam — real runner workers own their builds.** The installed Playwright and Vitest runners each execute a small authored fixture through the actual shared helper/adaptor path, using a private temporary root. Two cases assigned to the same worker reuse the source-CLI preparation and a same-version web bundle; a later worker owns a separate lifetime. Source CLI output starts a real product server, and the real web output is served/read, establishing that published results are usable. Within this same fixture, a bundle path naming an absent child makes the real copy fail after allocation of a per-launch runtime/home; those private outputs are removed, and a subsequent valid launch still uses the worker build. The missing path is fixture data. After per-case server cleanup the shared outputs still exist; after normal worker teardown they do not. No runner, builder, filesystem or server mechanism is mocked. This crosses each framework's teardown boundary under [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime), without a browser or an extra product assertion. [The real version-stamp seam](updates/version-advertisement.md#^t-web-stamp) supplies version breadth; this seam need not rebuild a matrix of versions. (test to be written) ^testing-t-build-worker-seam diff --git a/proofs/arch/themes/delivery.md b/proofs/arch/themes/delivery.md index 551b3752..b275fe91 100644 --- a/proofs/arch/themes/delivery.md +++ b/proofs/arch/themes/delivery.md @@ -60,7 +60,7 @@ The [visible application-theme seam](#^theme-delivery-t-visible-app) is covered - **Acceptance** (outer boundary: the built application in Chromium and Firefox against a real server and scratch data directory; fixture: an installed theme importing a nested stylesheet with relative day/night SVG URLs; no mocks or test hooks): selecting the theme resolves its registered `--app-wallpaper-image` relative to the declaring stylesheet, loads the correct asset through the application's composed `--app-wallpaper`, changes the image with appearance, and clears it when selecting None — *(covered by test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme-relative wallpaper URLs render through the built app token in Chromium and Firefox”)*. ^theme-delivery-t-wallpaper-url -- **Acceptance** (outer boundary: a real Chromium browser loads the built application and a live-canonical HTML artifact against a really-running server; a selected package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and temporary data directory are fixtures; no mocks or test hooks): changing only the nested stylesheet crosses the real `theme-changed` transport and application-link replacement, then changes imported computed presentation in both document classes without registry refresh — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-delivery-t-active-package-refresh +- **Acceptance** (outer boundary: a real Chromium browser loads the built application and a live-canonical HTML artifact against a really-running server; a selected package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and temporary data directory are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): changing only the nested stylesheet crosses the real `theme-changed` transport and application-link replacement, then changes imported computed presentation in both document classes without registry refresh — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”; built-application boundary: test to be written)*. ^theme-delivery-t-active-package-refresh - **Contract** (application theme-resource owner against a mocked DOM, forfeiting browser loading and execution to [the product browser acceptance](../../product/themes-and-appearance.md#^theme-ac-executable-browser)): a refresh creates one cache-busted classic external script in the document head at the connected server's absolute `/theme/main.js`; replacement detaches the prior element before appending the new one, clear removes it, and main-script operations neither replace nor reorder the stylesheet link or sandboxed frames — *(covered by `packages/web/test/theme.test.ts` “installs one cache-busted classic script from the connected server”, “removes the prior script before appending its replacement and clears it”, and “leaves stylesheet and sandboxed frame identity and ordering unchanged”)*. ^theme-delivery-t-app-script-module - **Contract** (renderer composition in a mocked DOM, with connection and application services replaced by controllable boundary mocks, forfeiting real connection event generation and browser resource loading to the browser and Electron assertions below): connection installs the application theme link before display state is ready; a selection change or same-ID resource-refreshing `theme-changed` event replaces it; a consent-membership-only event preserves it; and disconnect removes it — *(covered by test: `packages/web/test/renderer.test.ts` “ties the theme link to connection and theme events before display readiness” and “ties the theme script to confirmed consent, refresh, and connection lifecycle”)*. ^theme-delivery-t-app-link-renderer - **Contract** (application composition against controllable connection and application-service boundaries in a mocked DOM, forfeiting real event transport and script execution to [the product browser acceptance](../../product/themes-and-appearance.md#^theme-ac-executable-browser)): connection leaves the main script absent before confirmed display state; confirmed matching active-ID consent appends the first include only after first-party runtime setup; reconnect reevaluates confirmed state; opt-in injects without replacing the active stylesheet link or frame elements; active-package and registry events with unchanged selection and consent replace the stylesheet and main include; unrelated consent membership preserves all active resource identities; effective appearance changes preserve the stylesheet link and main include while frame replacement follows [the frame-appearance lifecycle](#^theme-delivery-t-frame-appearance-lifecycle); and disconnect removes them. Each path calls the one resource owner rather than using Settings registry state as an execution gate — *(covered by `packages/web/test/renderer.test.ts` “ties the theme script to confirmed consent, refresh, and connection lifecycle” and “reconciles theme frames across confirmed setup, refresh, selection, and connection state”)*. ^theme-delivery-t-app-script-renderer @@ -73,8 +73,8 @@ The [visible application-theme seam](#^theme-delivery-t-visible-app) is covered - **Contract** (application composition against controllable connection, display-state, and registry boundaries in a mocked DOM; deferred registry results and failures are authored outcomes; the mocks forfeit real transport and iframe replacement to [#^theme-delivery-t-frame-browser](#^theme-delivery-t-frame-browser)): selection change and same-ID active-package or registry refresh destroy both prior frames before resolving the current registry, then recreate only enabled destination frames. If initial or replacement lookup fails, both frames remain absent rather than retaining stale executable content; that failure does not interrupt the stylesheet, main-script, or application paths, and reconnect or a later selection, active-package, or registry trigger retries. A resolution or rejection from a superseded connection, theme ID, or lookup can neither install a frame nor remove frames installed by the current lookup. Consent-only events preserve both identities. Null selection and disconnect remove both frames; a registry refresh that disables either declaration leaves that frame absent while independently recreating the other enabled frame; and frame creation, replacement, or removal never requests a full application reload — *(covered by `packages/web/test/renderer.test.ts` “installs ready theme frames after deferred application mount”, “reconciles theme frames across confirmed setup, refresh, selection, and connection state”, “fails theme frame lookup closed while stylesheet, main script, and application continue”, “discards superseded theme frame resolutions and rejections”, and `packages/web/test/theme.test.ts` “remembers ordinary application focus and restores accidental current-frame focus”)*. ^theme-delivery-t-frame-lifecycle - **Contract** (the production application appearance-to-frame composition path in a mocked DOM; an effective root `data-theme` mutation, enabled registry snapshots, deferred fresh lookups, lookup failure, and rapid superseding mutations are authored outcomes; posing the root mutation and replacing the registry and DOM boundaries are mocks that forfeit the real appearance producer and event transport, script execution, and browsing-context replacement to [#^theme-delivery-t-frame-appearance-browser](#^theme-delivery-t-frame-appearance-browser)): under the [sandboxed-frame lifecycle](../../../specs/arch/themes/delivery.md#sandboxed-frames), observing an effective appearance change synchronously removes both current frames in the observer's callback turn before beginning the fresh registry lookup, preserves the active stylesheet and main-script elements without requesting application reload, then recreates every enabled frame for the new value. A failed initial or replacement appearance lookup leaves both absent; a later effective appearance change performs another fresh lookup and can restore enabled frames. A result superseded by a later appearance cannot install stale frames — *(covered by `packages/web/test/renderer.test.ts` “reconciles theme frames across confirmed setup, refresh, selection, and connection state”, “fails theme frame lookup closed while stylesheet, main script, and application continue”, and “discards superseded theme frame resolutions and rejections”)*. ^theme-delivery-t-frame-appearance-lifecycle - **Contract** (production host pointer forwarder against current frame-window and application-event boundary mocks in a browser DOM; the frame windows record `postMessage` and the event methods record interference, forfeiting real dispatch and hit testing to [#^theme-delivery-t-frame-browser](#^theme-delivery-t-frame-browser)): capture-phase listeners forward each trusted application-document `pointermove`, `pointerdown`, `pointerup`, `pointercancel`, and `click` once to each currently installed frame and never to a removed frame. The same internal forwarder accepts a source-authorized artifact input after the artifact owner maps it to application coordinates. Both paths map respectively to `television-theme-pointer-move`, `television-theme-pointer-down`, `television-theme-pointer-up`, `television-theme-pointer-cancel`, and `television-theme-pointer-click`; every message has only `type`, `clientX`, `clientY`, `button`, and `buttons`. Move sends `button: -1` and the input's current `buttons`; down sends the changed input `button` and post-transition `buttons` including the new press; up sends the changed input `button` and post-transition `buttons` excluding the release; click sends the clicked input `button` and `buttons: 0`; and cancel sends `button: -1` and `buttons: 0`, including for an authored cancel input reporting nonzero `buttons`. Each message uses the target frame's current `contentWindow` with destination `"*"`. Untrusted application events are ignored, messages are neither queued nor replayed, and forwarding never prevents default, stops propagation, or takes pointer capture. Any `message` event sourced from either current frame produces no application operation or reply regardless of payload, including an artifact-pointer lookalike — *(covered by `packages/web/test/theme.test.ts` “forwards the closed trusted pointer protocol without interfering or replaying” and “accepts no frame-originated command or reply path”, plus `packages/web/test/artifact-view.test.ts` “relays only current artifact pointer input to theme frames in application coordinates”)*. ^theme-delivery-t-frame-pointer -- **Acceptance** (outer boundary: the built application in Chromium against a really-running server and real package folders; current registry state, production host elements, public entry routes, the injected artifact bridge, and both theme iframe windows remain on the path; no mocks or test hooks): independently and jointly enabled entries execute automatically without main consent in separate `allow-scripts` opaque-origin documents, expose their cache-busted route through `document.currentScript.src`, cannot inspect application DOM, and contain load, syntax, and runtime failures. Real pointer move, down, up, cancel, and click input in the application document reaches both frames with the exact viewport coordinates and button fields while the same application sequence and click remain effective; a drag that produces no click forwards only its down, motion, and up transitions. Real move, down, up, and click input in a bridge-capable artifact also reaches both frames in application-viewport coordinates while the artifact's click remains effective. Ordinary application focus changes remain untouched. When either frame unexpectedly acquires focus, focus returns without scrolling to the last connected application element or, when unavailable, to `#app`; the production Settings popover remains operable in its document layer and an application modal dialog remains operable in the browser top layer — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. This is the same browser walk as [product acceptance](../../product/themes-and-appearance.md#^theme-ac-sandboxed-frames), not a second launch. ^theme-delivery-t-frame-browser -- **Acceptance** (outer boundary: the production Settings appearance control in the built Chromium application against a really-running server; a real package enabling both sandbox entries, entry-script startup probes, application-color probes visible through transparent regions of both frames, and a temporary data directory are fixtures; no mocks or test hooks): under the [sandboxed-frame appearance contract](../../../specs/arch/themes/delivery.md#sandboxed-frames), an effective light-to-dark appearance change records both old frame removals before either replacement, leaves no old-appearance frame, and reruns both entries. Each replacement document root carries `data-theme="dark"`; its used root scheme and its host element's used scheme are both dark. Rendered-pixel probes behind otherwise empty portions of the background and overlay frames remain visible, proving transparent iframe canvas composition rather than only transparent computed backgrounds — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. This is the same browser walk as [product frame-appearance acceptance](../../product/themes-and-appearance.md#^theme-ac-frame-appearance), not a second launch. ^theme-delivery-t-frame-appearance-browser +- **Acceptance** (outer boundary: the built application in Chromium against a really-running server and real package folders; current registry state, production host elements, public entry routes, the injected artifact bridge, and both theme iframe windows remain on the path; the real stable-front proxy supplies harness transport; no mocks or test hooks): independently and jointly enabled entries execute automatically without main consent in separate `allow-scripts` opaque-origin documents, expose their cache-busted route through `document.currentScript.src`, cannot inspect application DOM, and contain load, syntax, and runtime failures. Real pointer move, down, up, cancel, and click input in the application document reaches both frames with the exact viewport coordinates and button fields while the same application sequence and click remain effective; a drag that produces no click forwards only its down, motion, and up transitions. Real move, down, up, and click input in a bridge-capable artifact also reaches both frames in application-viewport coordinates while the artifact's click remains effective. Ordinary application focus changes remain untouched. When either frame unexpectedly acquires focus, focus returns without scrolling to the last connected application element or, when unavailable, to `#app`; the production Settings popover remains operable in its document layer and an application modal dialog remains operable in the browser top layer — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. This is the same browser walk as [product acceptance](../../product/themes-and-appearance.md#^theme-ac-sandboxed-frames), not a second launch. ^theme-delivery-t-frame-browser +- **Acceptance** (outer boundary: the production Settings appearance control in the built Chromium application against a really-running server; a real package enabling both sandbox entries, entry-script startup probes, application-color probes visible through transparent regions of both frames, and a temporary data directory are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): under the [sandboxed-frame appearance contract](../../../specs/arch/themes/delivery.md#sandboxed-frames), an effective light-to-dark appearance change records both old frame removals before either replacement, leaves no old-appearance frame, and reruns both entries. Each replacement document root carries `data-theme="dark"`; its used root scheme and its host element's used scheme are both dark. Rendered-pixel probes behind otherwise empty portions of the background and overlay frames remain visible, proving transparent iframe canvas composition rather than only transparent computed backgrounds — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. This is the same browser walk as [product frame-appearance acceptance](../../product/themes-and-appearance.md#^theme-ac-frame-appearance), not a second launch. ^theme-delivery-t-frame-appearance-browser - **Acceptance** (outer boundary: the real Electron app connected to a really-running server; an installed theme, its exact computed application-shell color, and an isolated Electron profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): selecting the installed theme renders its application style, and quitting and relaunching with the saved connection renders the same persisted theme without another selection — *(covered by test: `packages/desktop/test/e2e/webview-event-parity.test.ts` “renders the persisted app theme after quitting and relaunching”)*. ^theme-delivery-t-app-link-electron-startup - **Seam** (the production application theme-link module in Chromium against a really-running server on the app's origin and authored theme package resources; no mocks or test hooks): setup, selection, and watched-file refresh keep one cache-busted link last; each replaced link is detached; nested imports, font, and image resources load from the stable path; and appearance changes alter neither link identity nor request count — *(covered by test: `packages/web/test/e2e/appearance-delivery.test.ts` “theme link stays final and mode changes reuse its nested resources”)*. ^theme-delivery-t-app-link - **Acceptance** (outer boundary: the built application and a canonical-v2-styled HTML artifact in Chromium against a really-running server and its installed Clouds package; no mocks or test hooks): the marked app document receives the Clouds wallpaper through the stable route, the artifact keeps its canonical background without requesting or painting that wallpaper, and selecting `None` restores both bare baselines through the null theme while `/theme/theme.css` is empty — *(covered by `packages/web/test/e2e/appearance-delivery.test.ts` “Clouds targets the marked app without painting canonical artifacts”)*. ^theme-delivery-t-app-document diff --git a/proofs/index.md b/proofs/index.md index 8b6fb083..85f11e1d 100644 --- a/proofs/index.md +++ b/proofs/index.md @@ -28,7 +28,7 @@ Generated from the first-line description of every proof by `scripts/specs-index - `licensing.md` (24 lines) — *How the licensing machinery is proven: generator, gate and loader contracts over authored inputs, and seams through every real shipping build, the suite's inventory handoff and the committed asset manifest.* - `making-skills.md` (18 lines) — *How the promises in Making skills are proven.* - `node-versions.md` (18 lines) — *How the Node version contracts are proven: repository declarations, the advisory floor in the packed CLI package, and the toolchain checker at its two developer entry points.* - - `testing-policy.md` (27 lines) — *How worker-owned test builds are proven, and how testing discipline is reviewed.* + - `testing-policy.md` (25 lines) — *How worker-owned test builds are proven, and how testing discipline is reviewed.* - **artifact-frame/** - `artifact-bridge.md` (121 lines) — *How the promises in Artifact bridge are proven.* - `index.md` (24 lines) — *How the promises in Artifact frame (architecture) are proven.* diff --git a/proofs/product/artifacts.md b/proofs/product/artifacts.md index 6b1df013..8ddad115 100644 --- a/proofs/product/artifacts.md +++ b/proofs/product/artifacts.md @@ -51,8 +51,8 @@ Coverage relationship: fresh loading after a channel switch is crossed by [arch/ Browser demo mode ([#^af-demo-mode](../../specs/product/artifacts.md#^af-demo-mode)) is proven by two Chromium acceptance walks, with a routing contract for the breadth of which artifacts it reaches. Shared declaration for the two walks: the outer boundary is Chromium driving the built application and built unsupported-URL view against a really-running server launched with the server-home hook above; the external page is a two-page HTML fixture served by a local HTTP server on an origin other than the Television server's, never a public site; the external page artifact is created through the production API; the default motion override is the only mock. Playwright reads the cross-origin frame's document to observe which fixture page is shown. -- **Acceptance** (shared demo-mode walk): with no marker file, the external page artifact's frame shows the unsupported-URL error page. After the marker file is created and the app is reloaded, the frame's document is the fixture's first page at its own URL, the frame shows no Back or Forward control, and the artifact icon is a button named by [content.yml#return_aria_label](../../specs/ui/app/artifact-frame/content.yml#return_aria_label). After the marker file is deleted and the app is reloaded again, the frame shows the unsupported-URL error page and the icon is not a button — *(covered by test: `packages/web/test/e2e/browser-demo-mode.test.ts` "the home-directory marker turns direct external pages on and off at app load (^af-ac-demo-mode)")*. ^af-ac-demo-mode -- **Acceptance** (shared demo-mode walk, marker present): following the fixture's link shows its second page; pressing the artifact icon shows the first page again. Following the link again and reloading the app shows the first page. Following the link once more, switching to another channel, and returning shows the first page. Back and Forward remain absent throughout — *(covered by test: `packages/web/test/e2e/browser-demo-mode.test.ts` "the artifact icon, an app reload, and a channel round trip each return to the original page (^af-ac-demo-return)")*. ^af-ac-demo-return +- **Acceptance** (shared demo-mode walk): with no marker file, the external page artifact's frame shows the unsupported-URL error page. After the marker file is created and the app is reloaded, the frame's document is the fixture's first page at its own URL, the frame shows no Back or Forward control, and the artifact icon is a button named by [content.yml#return_aria_label](../../specs/ui/app/artifact-frame/content.yml#return_aria_label). After the marker file is deleted and the app is reloaded again, the frame shows the unsupported-URL error page and the icon is not a button — *(policy-grade test for the existing walk: `packages/web/test/e2e/browser-demo-mode.test.ts` "the home-directory marker turns direct external pages on and off at app load (^af-ac-demo-mode)"; built-application boundary: test to be written)*. ^af-ac-demo-mode +- **Acceptance** (shared demo-mode walk, marker present): following the fixture's link shows its second page; pressing the artifact icon shows the first page again. Following the link again and reloading the app shows the first page. Following the link once more, switching to another channel, and returning shows the first page. Back and Forward remain absent throughout — *(policy-grade test for the existing walk: `packages/web/test/e2e/browser-demo-mode.test.ts` "the artifact icon, an app reload, and a channel round trip each return to the original page (^af-ac-demo-return)"; built-application boundary: test to be written)*. ^af-ac-demo-return - **Contract** (the production artifact-frame view under jsdom with an authored external page artifact and a fake application service posing demo mode on — mocks that forfeit real document loading to [#^af-ac-demo-return](#^af-ac-demo-return); theme-change and artifact-content-change events are injected on that service): neither event reassigns the demo page's frame `src`, which stays the artifact's own URL — *(covered by test: `packages/web/test/artifact-view.test.ts` "browser demo mode artifact frame > leaves a demo-mode page's URL untouched through theme and content change events (^af-ac-demo-untouched)")*. ^af-ac-demo-untouched - **Contract** (the production routing functions `artifactRenderRoute` and `artifactDocumentRoute` over authored artifact records and recorded current URLs; no mocks): with demo mode on in a browser, `http:` and `https:` artifact URLs that are not Television artifact addresses route to a direct iframe at the artifact's own URL, including a URL on the Television host itself; a Television artifact address keeps the shared-artifact route both on another host and on the connected Television host; a non-web URL keeps the unsupported-URL page. Path and Markdown artifacts, a Television-served artifact whose recorded current URL is an external web page, and every artifact in the desktop app route identically with demo mode on and off — *(covered by tests: `packages/web/src/artifact-dispatcher.test.ts` "browser demo mode routing", all four cases)*. ^af-ac-demo-scope diff --git a/proofs/product/themes-and-appearance.md b/proofs/product/themes-and-appearance.md index ff2fdff2..91f09e86 100644 --- a/proofs/product/themes-and-appearance.md +++ b/proofs/product/themes-and-appearance.md @@ -41,16 +41,16 @@ Clouds' association with Television 1.3 composes with [product versioning's rele ### Acceptance criteria -- **Acceptance** (outer boundaries: real Settings controls over HTTP and websocket connections; the persistence and cross-browser `None`/Clouds walks load the built application, while the separate registry/control walk in `settings.test.ts` retains its development-module boundary; valid, invalid, and dot-prefixed theme folders and real temporary homes are fixtures; no mocks or test hooks): a fresh installation offers None plus the themes in [the bundled inventory](../../specs/ui/themes/bundled.yml); refresh shows valid packages and readable invalid-folder errors while every dot-prefixed folder is absent from options and errors; selection propagates to a second client; `None` selects the null theme and clears the active theme ID; the selected theme renders in Chromium and Firefox; and the server-confirmed selection survives a server restart and reconnection in the same browser page — *(covered by `packages/web/test/e2e/settings.test.ts` “settings controls cross the registry and confirmed display state” and `packages/web/test/e2e/theme-product-acceptance.test.ts` “the real dropdown renders its selected built-shell theme in Chromium and Firefox” and “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-ac-selection +- **Acceptance** (outer boundaries: real Settings controls over HTTP and websocket connections; the persistence and cross-browser `None`/Clouds walks load the built application, while the separate registry/control walk in `settings.test.ts` retains its development-module boundary; valid, invalid, and dot-prefixed theme folders and real temporary homes are fixtures; the persistence walk uses the real stable-front proxy as harness transport; no mocks or test hooks): a fresh installation offers None plus the themes in [the bundled inventory](../../specs/ui/themes/bundled.yml); refresh shows valid packages and readable invalid-folder errors while every dot-prefixed folder is absent from options and errors; selection propagates to a second client; `None` selects the null theme and clears the active theme ID; the selected theme renders in Chromium and Firefox; and the server-confirmed selection survives a server restart and reconnection in the same browser page — *(policy-grade tests for the existing walks: `packages/web/test/e2e/settings.test.ts` “settings controls cross the registry and confirmed display state” and `packages/web/test/e2e/theme-product-acceptance.test.ts` “the real dropdown renders its selected built-shell theme in Chromium and Firefox” and “theme settings persist rendered app and artifact outcomes through a built-server restart”; the persistence walk’s built-application boundary: test to be written)*. ^theme-ac-selection - **Acceptance** (outer boundary: Firefox drives the real Settings controls in the built application against a really-running production server and its real event transport; fixed-dark and light theme packages, a ready Markdown editor with authored contents, and a temporary home are fixtures; no mocks or test hooks): selecting the light theme changes both the shell and the same loaded Markdown editor from the fixed-dark presentation to the light presentation without replacing the editor document or losing its contents — *(covered by test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “Firefox refreshes a ready Markdown editor from a fixed-dark theme to light”)*. The same browser walk supplies the delivery evidence at [theme-delivery-t-markdown-refresh-firefox](../arch/themes/delivery.md#^theme-delivery-t-markdown-refresh-firefox). ^theme-ac-markdown-refresh-firefox -- **Acceptance** (outer boundary: two real Chromium application documents drive the built application's Settings controls against a running production server and its real event transport; main-enabled, unflagged, syntax-error, and uncaught-exception theme packages, a package-relative asset, an artifact document, and a temporary home are fixtures; no mocks or test hooks): before consent, the selected `enableMainJS: true` theme has no `main.js` include or request in either application document. Granting exact-ID consent runs its classic main script in both documents after the application runtime is available, exposes `document.currentScript.src`, resolves the package asset from that URL, survives a built-server restart, and leaves the artifact script-free. Appearance changes do not rerun it. An unrelated active-package edit, restoration after complete package-root loss, and server reconnection each rerun it; the missing-root interval executes nothing. An unflagged `main.js` executes nowhere. Separately consented syntax-error and uncaught-exception main scripts report browser errors and leave an ordinary Settings control operational — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-executable-browser +- **Acceptance** (outer boundary: two real Chromium application documents drive the built application's Settings controls against a running production server and its real event transport; main-enabled, unflagged, syntax-error, and uncaught-exception theme packages, a package-relative asset, an artifact document, and a temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): before consent, the selected `enableMainJS: true` theme has no `main.js` include or request in either application document. Granting exact-ID consent runs its classic main script in both documents after the application runtime is available, exposes `document.currentScript.src`, resolves the package asset from that URL, survives a built-server restart, and leaves the artifact script-free. Appearance changes do not rerun it. An unrelated active-package edit, restoration after complete package-root loss, and server reconnection each rerun it; the missing-root interval executes nothing. An unflagged `main.js` executes nowhere. Separately consented syntax-error and uncaught-exception main scripts report browser errors and leave an ordinary Settings control operational — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. ^theme-ac-executable-browser - **Acceptance** (outer boundary: the same two real Chromium application documents and running production server; a main-enabled theme whose script installs document-lifetime probes, a second theme, and persisted Settings-open state are fixtures; no mocks or test hooks): while the main include is active, withdrawing consent, selecting the second theme, and selecting `None` each reload both connected application documents, clear the prior script's document-lifetime effects, and start under the confirmed destination state. Only a document whose Settings popover was open reopens it; its URL marker is consumed before reopening and causes no later reload. After selecting the second theme, selecting the still-consented main-enabled theme again reinstalls its include and executes it without another consent grant. Changing themes while no main include is installed remains in place — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable theme reset reloads connected Chromium documents under confirmed state”)*. ^theme-ac-executable-reset -- **Acceptance** (outer boundary: the same real Chromium application and running production server; a package independently enabling both sandbox entries, a destination theme, syntax-error and uncaught-exception iframe entries, Settings and application modal-dialog controls, and pointer-observation probes are fixtures; no mocks or test hooks): without main-page consent, the background and foreground entries execute in separate opaque-origin `allow-scripts` frames at their specified layers and cannot access the application DOM. Theme selection and an unrelated active-package edit replace both frame documents; package-root loss replaces them with empty entries, restoration executes fresh frames, and registry refresh that disables either declaration removes only that frame. Syntax and uncaught runtime errors stay in their affected frames while the application operates. Trusted pointer move, down, up, cancel, and click events in the application document reach both frames with viewport coordinates and button state while the same underlying application sequence, including click, remains effective; a drag that produces no click forwards its down, movement, and up but no click. Pointer move, down, up, and click inside the bridge-capable artifact reach both frames in application-viewport coordinates while the artifact's own click remains effective. Unexpected focus returns first to the last connected application element and then to `#app` when that element is unavailable. The foreground layers do not prevent using the production Settings popover in its document layer or an application modal dialog in the browser top layer — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-sandboxed-frames -- **Acceptance** (outer boundary: the production Settings appearance control in a real Chromium application against a running production server; a real active package enabling both sandbox entries, entry-script startup probes, visible application probes behind transparent regions of both frames, and a temporary home are fixtures; no mocks or test hooks): under the [frame lifecycle](../../specs/product/themes-and-appearance.md#^theme-iframe-lifecycle), an effective light-to-dark appearance change removes both old frame documents before installing replacements, leaves no old-appearance frame, and reruns both entries in documents whose root `data-theme` is `dark`. Each replacement host and document root has a used dark scheme, and the application probes remain visibly unchanged through the frames' transparent regions — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-frame-appearance -- **Acceptance** (outer boundary: a real Chromium browser loads the built application and a live-canonical HTML artifact against a running built server and its real `/events` transport; a real active package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and a temporary home are fixtures; no mocks or test hooks): changing only the nested stylesheet updates the application and artifact presentation; moving the complete active package away produces their foundation and canonical baselines while the exact selected theme ID remains confirmed; restoring it at the same path restores both presentations and keeps that ID, without registry refresh or another selection — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-ac-active-package +- **Acceptance** (outer boundary: the same real Chromium application and running production server; a package independently enabling both sandbox entries, a destination theme, syntax-error and uncaught-exception iframe entries, Settings and application modal-dialog controls, and pointer-observation probes are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): without main-page consent, the background and foreground entries execute in separate opaque-origin `allow-scripts` frames at their specified layers and cannot access the application DOM. Theme selection and an unrelated active-package edit replace both frame documents; package-root loss replaces them with empty entries, restoration executes fresh frames, and registry refresh that disables either declaration removes only that frame. Syntax and uncaught runtime errors stay in their affected frames while the application operates. Trusted pointer move, down, up, cancel, and click events in the application document reach both frames with viewport coordinates and button state while the same underlying application sequence, including click, remains effective; a drag that produces no click forwards its down, movement, and up but no click. Pointer move, down, up, and click inside the bridge-capable artifact reach both frames in application-viewport coordinates while the artifact's own click remains effective. Unexpected focus returns first to the last connected application element and then to `#app` when that element is unavailable. The foreground layers do not prevent using the production Settings popover in its document layer or an application modal dialog in the browser top layer — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. ^theme-ac-sandboxed-frames +- **Acceptance** (outer boundary: the production Settings appearance control in a real Chromium application against a running production server; a real active package enabling both sandbox entries, entry-script startup probes, visible application probes behind transparent regions of both frames, and a temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): under the [frame lifecycle](../../specs/product/themes-and-appearance.md#^theme-iframe-lifecycle), an effective light-to-dark appearance change removes both old frame documents before installing replacements, leaves no old-appearance frame, and reruns both entries in documents whose root `data-theme` is `dark`. Each replacement host and document root has a used dark scheme, and the application probes remain visibly unchanged through the frames' transparent regions — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. ^theme-ac-frame-appearance +- **Acceptance** (outer boundary: a real Chromium browser loads the built application and a live-canonical HTML artifact against a running built server and its real `/events` transport; a real active package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and a temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): changing only the nested stylesheet updates the application and artifact presentation; moving the complete active package away produces their foundation and canonical baselines while the exact selected theme ID remains confirmed; restoring it at the same path restores both presentations and keeps that ID, without registry refresh or another selection — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”; built-application boundary: test to be written)*. ^theme-ac-active-package - **Acceptance** (outer boundary: the real Electron app and another client connected to a running server; a theme enabling main, background, and foreground entries, a second theme, an artifact webview, and an isolated Electron profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): before consent the main script does not execute, while both sandboxed frames are created and execute in the application. Pointer move, down, up, and click inside the ordinary HTML artifact webview reach both theme frames in application-viewport coordinates while the artifact's click remains effective. Granting consent through Settings executes `main.js` in the application renderer but no theme JavaScript in the artifact webview, and the renderer exposes the same Node-facing globals and preload bridge before and after activation. Consent survives application restart; remote main opt-out and remote selection of the second theme each reload the application document, clear its document-lifetime probe, and apply confirmed state. A remote same-theme active-package event replaces both frame documents without itself causing a full application reload — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “executes an executable theme in Electron and resets on remote consent and selection changes”)*. ^theme-ac-executable-electron - **Acceptance** (outer boundary: the real Electron app connected to a running server; an installed theme and isolated Electron profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): the selected application theme renders again after quitting and relaunching through the saved connection — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “renders the persisted app theme after quitting and relaunching”)*. ^theme-ac-selection-electron -- **Acceptance** (outer boundary: the built application in a real Chromium browser against a really-running server; one adaptive package and one fixed-dark package, local Television-managed HTML and markdown artifacts, a third-party URL artifact, both sandboxed entries with startup probes, dedicated computed-style probes, controlled system appearance, and a real temporary home are fixtures; no mocks or test hooks): under the adaptive theme, `Adapt to system`, `Light`, and `Dark` produce their expected effective appearance in the application and applicable artifacts. Under the fixed-dark theme, selecting and persisting an opposing Light preference leaves the application root, foundation mode values and native scheme, both artifact classes, and both frame host/document pairs dark; it does not reload app or artifacts, and it keeps the existing frame documents because effective appearance did not change. The same fixed results survive a built-server restart. Returning to the adaptive theme applies the latest Light preference; that effective change replaces and reruns both frames while leaving their transparent composition intact — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart” and “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-appearance +- **Acceptance** (outer boundary: the built application in a real Chromium browser against a really-running server; one adaptive package and one fixed-dark package, local Television-managed HTML and markdown artifacts, a third-party URL artifact, both sandboxed entries with startup probes, dedicated computed-style probes, controlled system appearance, and a real temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): under the adaptive theme, `Adapt to system`, `Light`, and `Dark` produce their expected effective appearance in the application and applicable artifacts. Under the fixed-dark theme, selecting and persisting an opposing Light preference leaves the application root, foundation mode values and native scheme, both artifact classes, and both frame host/document pairs dark; it does not reload app or artifacts, and it keeps the existing frame documents because effective appearance did not change. The same fixed results survive a built-server restart. Returning to the adaptive theme applies the latest Light preference; that effective change replaces and reruns both frames while leaving their transparent composition intact — *(policy-grade tests for the existing walks: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart” and “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. ^theme-ac-appearance - **Acceptance** (outer boundary: the real Electron app connected to a really-running server; one adaptive and one fixed-dark installed package, an existing Television-managed artifact webview, a third-party URL artifact webview, native-appearance observation, computed-style probes, and an isolated profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): the adaptive theme carries `system`, `light`, and `dark` through `nativeTheme` to both retained webviews and their media-query results. Under the fixed-dark theme, an opposing Light preference persists while the application, `nativeTheme`, native-surface appearance source, and both current webviews remain dark and loaded. Returning to the adaptive theme applies that latest Light preference; the third-party URL webview stays loaded, while the Television-managed artifact follows its ordinary theme-selection reload lifecycle — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “applies manifest-resolved appearance to Electron and retained webviews”)*. This is the same Electron walk as [desktop appearance acceptance](../arch/desktop/appearance.md#^desktop-appearance-t-electron), not a second launch. ^theme-ac-appearance-electron - **Acceptance** (outer boundary: a real browser loads artifacts from a really running production `Server`; the committed old-style v1 document, an authored v2 control document, fresh production canonical tree, temporary storage, and installed loud unscoped theme are fixtures; no mocks or test hooks): the v1 artifact loads its wrapper, base, components, and font but makes no theme request, renders the fixed light canonical probe, and does not render the loud theme value; the v2 control requests the theme and renders its loud value and dark scheme — *(covered by `packages/server/test/e2e/canonical.spec.ts` “frozen v1 stays fixed light while live v2 receives the active theme”)*. ^theme-ac-frozen-v1 - **Acceptance** (outer boundary: a spawned built CLI server against a real fresh temporary home; the packaged bundled-theme tree is the fixture; no mocks or test hooks): first boot installs every bundled theme byte-for-byte, selects Clouds from an unset theme, exposes its exact stylesheet and wallpaper assets, and records the one-time decision — *(covered by `test/node/theme-packaging.test.ts` “Built CLI installs every packaged theme and initially selects Clouds”)*. ^theme-ac-bundled diff --git a/specs/arch/test-runner/blaxel-testshards.md b/specs/arch/test-runner/blaxel-testshards.md index 8b933ef9..02e46df0 100644 --- a/specs/arch/test-runner/blaxel-testshards.md +++ b/specs/arch/test-runner/blaxel-testshards.md @@ -169,6 +169,6 @@ Under the testing policy's [rule for integration against paid services or comple A change to pool provisioning or runtime selection requires the live run to use freshly provisioned workers, since `ensure` refuses a populated pool ([#Pool](#Pool)). A dedicated pool may supply this evidence without deleting shared `poc` workers. -Recovery and cleanup experiments use disposable owned processes on a dedicated pool, never intentionally contaminating shared workers. Temporary-file placement needs live observation of Node-created paths and Chromium shared-memory descriptor paths under the attempt's `TMPDIR`. +Recovery and cleanup experiments use disposable owned processes on a dedicated pool, never intentionally contaminating shared workers. Automated qualification uses the runner's [Blaxel harness placement](./test-runner.md#^blaxel-harness-placement) and the registry's explicitly selected experiment surfaces, excluded from `all` and ordinary CI. Temporary-file placement needs live observation of Node-created paths and Chromium shared-memory descriptor paths under the attempt's `TMPDIR`. A change to repository selection, or to how a worker's checkout moves between repositories, requires live runs from checkouts of two repositories on the same pool, including a run on workers whose checkout last served the other repository. diff --git a/specs/arch/test-runner/test-registry.md b/specs/arch/test-runner/test-registry.md index 3a6737d6..7308b283 100644 --- a/specs/arch/test-runner/test-registry.md +++ b/specs/arch/test-runner/test-registry.md @@ -92,7 +92,9 @@ The registered suites: - `daemon-acceptance` → `kind:daemon-acceptance` - `all` → `kind:unit`, `kind:e2e` -`all` is the broad suite and is deliberately unit + non-agent e2e: agent, experiment, live PostHog roundtrip, and production-identity daemon acceptance surfaces are not part of `all`. Agent, telemetry, and daemon acceptance surfaces must be selected explicitly with their named suites. The experiment suite contains two placement classes. Lifecycle-fault experiments carry `isolated-github-only`, so selection alone cannot bypass their execution-placement guard. `experiment:dynamic-services` is a normal runnable service fixture without that tag; its protection is exclusion from `all`, so it runs only when selected explicitly. `telemetry-posthog-roundtrip` owns the live telemetry ↔ PostHog integration surface that requires secrets. `daemon-acceptance` owns the host-mutating packed-CLI and production-service lifecycle surface. Both are excluded from CI and broad verification. Build-config-integrity coverage is a normal unit surface, `unit:build-config`, so it runs in `all` and CI without secrets. +`all` is the broad suite and is deliberately unit + non-agent e2e: agent, experiment, live PostHog roundtrip, and production-identity daemon acceptance surfaces are not part of `all`. Agent, telemetry, and daemon acceptance surfaces must be selected explicitly with their named suites. Lifecycle-fault experiments carry `isolated-github-only`, so selection alone cannot bypass their execution-placement guard. `experiment:dynamic-services` is a normal runnable service fixture without that tag; its protection is exclusion from `all`, so it runs only when selected explicitly. `telemetry-posthog-roundtrip` owns the live telemetry ↔ PostHog integration surface that requires secrets. `daemon-acceptance` owns the host-mutating packed-CLI and production-service lifecycle surface. Both are excluded from CI and broad verification. Build-config-integrity coverage is a normal unit surface, `unit:build-config`, so it runs in `all` and CI without secrets. + +Live Blaxel harness qualification uses two separately owned `kind: "experiment"` surfaces: `experiment:blaxel-harness` is the automated driver, and `experiment:blaxel-harness-fixtures` contains the native fixtures it dispatches, including deliberate failures and recovered retries. The fixture surface carries the selection tag `blaxel-harness-fixture` and never dispatches another run. Both are excluded from `all` and ordinary CI; explicit selection cannot recursively include the driver in its fixture runs. Their execution placement is owned by [Blaxel harness qualification](./test-runner.md#Blaxel harness qualification). ^blaxel-harness-surfaces ## Execution groups @@ -178,4 +180,3 @@ Registry discovery and validation scan only the *canonical include roots* `packa - a surface whose `roots` or `excludeRoots` entry does not exist, - a service with a duplicate `id`, a missing Vite config, an unsupported `kind`, an invalid `publishUrlEnv` name, or a `publishUrlEnv` duplicated within its surface, - an unregistered runner config file found under the include roots — any file matching `vitest.config.ts`, `vitest.e2e.config.ts`, or `playwright.config.ts` that no surface declares. - diff --git a/specs/arch/test-runner/test-runner.md b/specs/arch/test-runner/test-runner.md index 17149bfc..15e4e252 100644 --- a/specs/arch/test-runner/test-runner.md +++ b/specs/arch/test-runner/test-runner.md @@ -144,6 +144,12 @@ The surface builds the publishable CLI, packs it, and force-installs that tarbal Persisted serve rejects port `0`, so the surface cannot keep a kernel-owned port reservation while handing a stable number to the service's config file. For each installation it creates a temporary [Television home](../../product/cli.md#^cli-home), binds loopback port `0`, reads the assigned nonzero port, closes the reservation, writes that port with `tv --home config set port `, and immediately installs with `tv --home serve --persist` and probes the daemon on that number. This is a narrow declared exception to [the normal continuous-reservation rule](#^test-dynamic-ports). `TV_DAEMON_TEST_DIRTY_START=1` asks the surface to create a production-identity installation before its foundation bulldozer; this is a test-harness mode for proving convergence from prior service residue, and it remains protected by `TV_DAEMON_TEST_HOST=1`. +## Blaxel harness qualification + +Live tests of the Blaxel harness use the opt-in [driver and fixture surfaces](./test-registry.md#^blaxel-harness-surfaces). Run the automated driver with `npm test -- local --surface experiment:blaxel-harness` on a dedicated credentialed Blaxel coordinator sandbox outside the worker pool, using a committed and pushed checkout. It must not run on a shared developer host or a leased shard worker. It owns a disposable private pool and uses canonical Blaxel commands against that pool; it never selects itself for remote execution. Ordinary repository contracts and Linux process/filesystem seams remain ordinary repository tests and need no live-provider credentials. ^blaxel-harness-placement + +The driver selects only the committed fixture surface for worker runs: `--tag blaxel-harness-fixture` delegates a planned `--suite selection --surfaces experiment:blaxel-harness-fixtures` run; `--surface experiment:blaxel-harness-fixtures` delegates a targeted run. Workers load these fixtures from the committed registry, without a forwarded replacement-registry hook. Deliberate fixture failures remain failed native results; the driver asserts their expected outcomes and retained evidence. The driver cleans up its coordinators and private pool when it finishes, retaining reports and identifying any cleanup it could not complete. + ## Retries Non-unit Vitest and Playwright surfaces retry twice at the runner level by default, including targeted file/grep/surface runs; unit surfaces receive no runner-level retries. `--retries ` overrides the non-unit count and is ignored (with a notice) for a unit-only selection. This is the runner's blanket retry budget; the per-test flaky annotation that opts a single case into retries is owned by [flaky-tests.md](./flaky-tests.md). diff --git a/specs/index.md b/specs/index.md index 70ca30cd..4b647979 100644 --- a/specs/index.md +++ b/specs/index.md @@ -96,8 +96,8 @@ Generated from the first-line description of every spec by `scripts/specs-index. - `preflight.md` (92 lines) — *The checks the test runner runs before tests: local capability checks for the selected surfaces, and the git-safety and provider-auth checks that gate a committed-revision remote run.* - `reporting.md` (426 lines) — *The run directory, normalized reports, resource diagnostics, timing events, and CI run artifacts test runs produce.* - `sharded-execution.md` (343 lines) — *The shared remote execution model: deterministic duration-aware plans, per-worker execution, provider reports, retries, and normalization.* - - `test-registry.md` (181 lines) — *The test registry: how `test.config.mjs` declares the surfaces, suites, and execution groups the test runner selects, owns files against, and validates.* - - `test-runner.md` (262 lines) — *The canonical `npm test` and `npm run verify` command surface: how a caller selects tests, picks where they run, and runs them, with the guardrails that keep broad runs honest.* + - `test-registry.md` (182 lines) — *The test registry: how `test.config.mjs` declares the surfaces, suites, and execution groups the test runner selects, owns files against, and validates.* + - `test-runner.md` (268 lines) — *The canonical `npm test` and `npm run verify` command surface: how a caller selects tests, picks where they run, and runs them, with the guardrails that keep broad runs honest.* - **themes/** - `authoring.md` (204 lines) — *Theme-authoring guidance: the self-contained theming document bundled inside the `television` skill, the authoritative UI material it carries, and the workflow it gives an agent.* - `bundled-installation.md` (77 lines) — *Bundled theme installation: validated package assets, CLI path resolution, minimum-version installation and replacement with backup copies, and the one-time default-theme selection.* From 70f27ccb27423d6705d8b3436973965ca35f4279 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:23:02 +0000 Subject: [PATCH 08/64] Plan resource cleanup slices after proof convergence --- .../test-harness-resource-cleanup-plan.md | 92 +++++++++++++++++++ ...arness-resource-cleanup-proof-review-r2.md | 61 ++++++++++++ .../test-harness-resource-cleanup-proposal.md | 8 +- ...arness-resource-cleanup-review-response.md | 20 +++- proofs/arch/test-runner/blaxel-testshards.md | 4 +- proofs/arch/test-runner/reporting.md | 2 +- proofs/arch/test-runner/test-registry.md | 2 +- specs/arch/test-runner/test-runner.md | 4 +- 8 files changed, 182 insertions(+), 11 deletions(-) create mode 100644 docs/working/test-harness-resource-cleanup-plan.md create mode 100644 docs/working/test-harness-resource-cleanup-proof-review-r2.md diff --git a/docs/working/test-harness-resource-cleanup-plan.md b/docs/working/test-harness-resource-cleanup-plan.md new file mode 100644 index 00000000..20fa1fba --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-plan.md @@ -0,0 +1,92 @@ +# Test-harness resource cleanup: implementation plan + +Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: ready for independent plan review; implementation has not begun. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. + +## Slice judgment + +Use three slices, each with tests and implementation reviewed together. Helper ownership and shipped-asset loading can be completed independently of the remote coordinator. Diagnostic capture has a separate native-runner/reporting boundary. Lease acquisition, dispatch registration, recovery, scratch cleanup, pool mutation and result retrieval must remain together: splitting that lifecycle would leave intermediate combinations whose safety is hard to assess. + +The order is helper lifetime, diagnostics, then worker lifecycle and live qualification. The last slice uses the earlier diagnostics and helper fixes on real workers, so the remote acceptance runs prove their composition once. Each slice receives independent implementation review, followed by review and full verification of the integrated result. This is one PR; these are review boundaries, not separate releases. + +The specs and proofs remain the source of requirements. Paths below identify the current implementation areas, not a fixed inventory of files that may change. No production application behavior, retry default, standard shard count or browser version changes belong here. + +## Execution and evidence + +Nothing in this plan runs tests, builds, lint, type checks, browsers or servers on the shared development host. Edit, commit and push each red or green revision there. Every execution uses that pushed revision on Blaxel through the canonical runner. A failed import, missing dependency, unavailable credential or setup failure is not the intended red result: establish that the narrow test reaches the missing behavior before counting it as red. + +Use the named `tv-cleanup-coordinator-rose` sandbox outside all worker-pool labels and private pool `rose-cleanup`, with workers `tv-testshard-x64-rose-cleanup-01` and `-02`. Inventory, credentials, architecture, runtime and available capacity must be established before execution; the names are a plan, not evidence that hosts exist. The coordinator needs the repository's selected Node, dependencies and the local preflight capabilities required by the selected driver. The proposal owns ephemeral credential injection and credential-free logs. No test or experiment in this plan uses shared `poc` workers. + +Run ordinary focused tests from the coordinator as `npm test -- blaxel --pool rose-cleanup --file `, adding `--grep ` when narrowing a known failure. Surface expansion also names `--pool rose-cleanup`. Keep the default native and infrastructure retry budgets. Use a shortened existing recovery interval only for the declared abandoned-run experiments; it changes no production default. Ensure the canonical CLI forwards that existing coordinator option when implementing the live driver. + +Run the live driver on the coordinator as `npm test -- local --surface experiment:blaxel-harness`. It calls the canonical Blaxel runner and pool manager for `rose-cleanup`; it is never itself a worker selection. Do not run the entire experiment suite or any isolated-GitHub-only lifecycle fixture. Before its fresh-provisioning acceptance, retrieve earlier private-run results and delete the now-idle private pool through the pool manager. The driver owns its coordinator subprocesses and pool cleanup, and reports any ownership or cleanup it cannot confirm. + +For each slice, retain the red and green commit IDs, exact commands, canonical run-directory paths, relevant assertion failures and recovered retries in the working response. After a targeted pass, expand only to affected files/surfaces. A broad failure returns to the failing case or file; a full gate is not an iteration probe. Deliberately failing native fixtures keep their failed reports even when the driver correctly passes by observing those failures. + +## Slice 1 — worker-owned builds and built application walks + +**Result.** The source CLI and versioned web outputs belong to a test worker, remain reusable across its cases, and are removed at teardown. Failed preparation and failed per-launch copying leave no private output. Five selected browser cases load shipped assets while keeping their product assertions. + +**Implementation area.** `test/helpers/product-server.ts`, `versioned-web-bundle.ts`, the shared Playwright registration and the necessary Vitest lifecycle adapters. Reuse a small common build-lifetime owner for these two actual callers. Preserve version-keyed reuse and the web builder's existing serialization. Register ownership before preparation can fail and publish a result only after preparation succeeds. Per-case product-server disposal must not remove the worker cache. Real framework teardown, rather than a test-only disposal call, must close the lifetime. + +Convert only the persistence and executable-lifecycle cases in `packages/web/test/e2e/theme-product-acceptance.test.ts`, and the three cases sharing `packages/web/test/e2e/browser-demo-mode.test.ts`'s page setup. Use the existing stable-front proxy for restart continuity and authentication; retain backend port `0`. Serve the built unsupported-URL view in demo mode. Preserve PR #9's reset walk, the separate Settings development-module walk, other deferred callers and main's demo fixture teardown correction. + +**Red/green and review criteria.** Implement the [build-lifetime contract and real worker seam](../../proofs/arch/testing-policy.md#Worker-owned builds). Begin with narrow red cases for early publication/failed preparation and outputs surviving worker teardown. Contract builders are the declared substitutes; the Playwright and Vitest seam must build real CLI/web output, use it, reuse it within a worker, retain it after case disposal and remove it after worker teardown. Exercise the absent-bundle copy failure within that same fixture, then a valid launch using the reusable build. Put cheap contracts in the ordinary repository suite and the real build/server crossing in the existing Node e2e surface, with fixture configs owned through the existing nested-runner arrangement. + +For the five converted walks, make their built-loading boundary observable in the existing runs before changing page setup, then preserve all existing product assertions through the adaptation. Retain a request inventory showing the built page rather than the development-module graph; this adds no separate browser launch or source-text compliance scan. Confirm same-origin authenticated operation and restart/reload continuity. Grade the affected [theme](../../proofs/product/themes-and-appearance.md), [delivery](../../proofs/arch/themes/delivery.md), [artifact](../../proofs/product/artifacts.md) and [bridge](../../proofs/arch/artifact-frame/artifact-bridge.md) citations only after the real adapted walks pass. + +Expand from the new helper tests and the five cases to both affected browser files, then the owning browser surfaces and directly affected helper callers where the change predicts impact. Preserve existing per-case process cleanup. This slice is green for its assigned obligations; killed-worker scratch recovery remains explicitly assigned to slice 3. + +## Slice 2 — bounded diagnostic capture and report transport + +**Result.** Surface parents retain start/end resource observations and bounded failure evidence independently of the native test child's survival. Sidecars keep native outcomes, recovered retries and attempt identities intact through the reporting pipeline. + +**Implementation area.** A shared collector/writer used by the real surface parent, planned shard execution and the targeted Blaxel path; `scripts/test/reporting.mjs`, provider normalization/output aggregation, their declarations, and the coordinator's artifact download. Keep the existing native reporters and result tree. Add `resourceDiagnosticsPaths` only for retained files. A common writer handles the stated sidecar and attempt budgets, including receipts later produced by slice 3; do not add a sampler, archive or retention daemon. + +**Red/green and review criteria.** Discharge [resource bounds](../../proofs/arch/test-runner/reporting.md#^reporting-t-resource-bounds), [parent capture](../../proofs/arch/test-runner/reporting.md#^reporting-t-resource-parent), and [retained links](../../proofs/arch/test-runner/reporting.md#^reporting-t-resource-links). Start with narrow writer/normalizer red assertions, then cross the actual parent/native-runner boundary on Linux. Both planned and targeted entry points must retain a baseline before preparation, finalize after teardown, and keep a missing final sample explicit after parent interruption. Inspect real probe sources/units without asserting transient host totals or manufacturing an OOM. + +Use the reviewed injected clock, probe readings and write failures for unavailable/reset/oversized cases. Preserve 64 KiB of kernel lines at most, 128 KiB per sidecar and 8 MiB per shard attempt including receipts; preserve compact observations before optional excerpts. Exclude secrets, arbitrary environments, arguments, bodies and query strings. Diagnostic failure warns without changing the native result. Earlier attempt links survive a later successful infrastructure attempt, and missing native output cannot create a passing selected-surface result. + +The ordinary parent/child seams use the existing guarded local registry hook inside their Blaxel-hosted execution. It is never forwarded to workers. These tests use real native reports, not fabricated replacements. Expand to the affected reporting, shard-worker, provider and native-recorder tests after the new focused cases pass. + +**Explicit remaining baseline.** The real Blaxel download/reuse and abandoned-run receipt evidence is not discharged here. `^reporting-t-resource-live` stays marked pending and belongs to slice 3's shared live runs; no second paid-service driver or diagnostic-only crash scenario is added. The current lease/download race is also still present until slice 3. Do not describe this checkpoint as safe for concurrent worker reuse or as a completed contribution. + +## Slice 3 — exclusive worker lifecycle and live qualification + +**Result.** One worker remains exclusively owned from acquisition through confirmed process teardown and retained results. Lost renewal permits automatic cleanup before reuse. Scratch and known leftovers are removed only under the reviewed ownership conditions. Pool operations follow that same exclusion. Live qualification proves the whole path, including slice 2's diagnostics. + +**Implementation area.** `scripts/run-blaxel-testshards.mjs`, `manage-blaxel-testshards.mjs`, their lease/discovery/dispatch helpers, the existing process owner/audit mechanisms, and the shared/targeted runner paths. Keep acquisition and entry checks usable before checkout; they cannot depend on code in a checkout another run may still own. Preserve the outer shard-attempt token across nested surface owners. A short OS-released guard serializes worker-local ownership changes; it is never held across report downloads. + +Keep acquisition, renewal, dispatch registration, normal release, interruption and recovery in one lifecycle. Renew through setup, execution, teardown and retrieval; remove shell-exit unlock. Normal release follows confirmed descendants and verified downloads. The one expired-renewal procedure stops attributed owners, audits, records abandoned report loss and only then reuses the worker. Preserve the reviewed legacy behavior and fail-closed cases. Pool deletion refuses valid leases and removes dispatch eligibility before deletion; fresh provisioning publishes eligibility only after successful verification. No new service, caller-attestation system or mixed-version framework is needed. + +Give each shard attempt its specified private scratch and `TMPDIR`. Route uploads, plan, askpass and setup sidecars there, finalizing evidence outside it before removal. Pre-run cleanup removes only the two known direct `/tmp` prefixes and non-current attempt directories after the lease and clean owner audit. Preserve unrelated files, links' outside targets, current scratch and caches; reject unsafe roots/mounts. Stop processes before post-run removal and retain evidence/ownership when cleanup is uncertain. + +**Red/green and review criteria.** First establish red at the [lease decision and ordering contracts](../../proofs/arch/test-runner/blaxel-testshards.md#Lease, recovery, and scratch), then implement the lifecycle and cross the real Linux guard/registration and filesystem seams. Cover competing acquisition, old-owner mutation, delayed entry, nested tokens, token/PID-start-validated termination, incomplete records, cleanup errors and permitted legacy behavior at their reviewed shapes. The ordinary guard seam restricts its audit to fixture tokens as declared; it supplies no live-provider credit. Replace the existing TTL-only assertions rather than leaving tests that endorse the superseded release behavior. Test pool publication/deletion ordering and discovery of unavailable partial workers with the declared SDK substitutions before the live pool crossing. + +Register `experiment:blaxel-harness` and `experiment:blaxel-harness-fixtures`, giving the fixture surface at least two native files. Discharge the [registry selection contract](../../proofs/arch/test-runner/test-registry.md#^registry-t-blaxel-harness-selection): neither surface enters `all`/ordinary CI, the fixture tag selects only fixtures, and the driver never dispatches itself. The fixtures include failed and recovered native outcomes, bounded owned child processes and the small real Chromium page. The driver uses the reviewed pre-download rendezvous; ordinary runs never wait there. + +After the focused contract/seam tests are green, run the real driver from the dedicated coordinator on the pushed revision. These observations share one small fresh pool: + +- Planned two-shard execution and a targeted run cross acquisition, checkout, setup, native execution and actual transport. Retain uploaded-script paths, `temporary-cleanup` phase/counts, native outcomes and diagnostics. A competing run and pool deletion are refused while download is gated and renewal continues. After retrieval, scratch is gone, allowed legacy residue is gone, unrelated sentinels/caches remain, and normal reuse works. +- Kill the actual lease-renewing coordinator in the two reviewed scenarios: an active supervisor/owned descendant, and quiet work before retrieval. After the deliberately shortened recovery interval, the next acquisition terminates the old owned processes, confirms disuse, retains the receipt and reuses the worker. No discarded report can become a pass. Do not substitute killing only the shard or merely stopping a mocked renewal callback. +- Inspect real Node-created paths and Chromium shared-memory descriptors under the attempt's `TMPDIR`, including unlinked files. Record the browser version and executable/cache path. This verifies placement without claiming peak memory or recreating exhaustion. +- The failed fixture stays failed; the recovered fixture retains all native attempts. Downloaded diagnostic links survive scratch removal, carry the correct identities and expose unavailable OOM evidence honestly. Delete the private pool through the real manager after reports are retained, confirming the provider metadata/deletion boundary. + +These discharge `^blaxel-t-retrieval`, `^blaxel-t-recovery-live`, `^blaxel-t-tmpdir-live`, `^blaxel-t-pool-live` and `^reporting-t-resource-live`. The source-level registry contract and SDK mocks do not substitute for any of them. All temporary baselines from slices 1 and 2 are resolved before this slice is called complete. + +## Integrated result and final validation + +After slice convergence, independently review the integrated specs, proofs, tests and code. Verify every new pending citation is either discharged with its real evidence or reported as unfinished; intentional fixture failures remain distinguishable from test-runner failures. Preserve the separate Settings development path, PR #9 reset, normal stale-owner protection and isolated-GitHub fixture restrictions. + +Fetch the PR target, integrate it under the logical-merging workflow if needed, then commit and push the tree before the qualifying gate. Any target movement requires validation of the updated tree. Prepare fresh private workers after the live driver's pool-deletion case and run on `tv-cleanup-coordinator-rose`: + +```sh +npm run verify -- blaxel --pool rose-cleanup --shards 2 --allow-shard-count-override 1 +``` + +This runs every `all` shard index plus lint, type checking and package-manifest validation on Blaxel. The [publication-qualification rule](../../specs/arch/test-runner/test-runner.md#^test-publication-qualifying) permits a different partition count when the complete suite runs; it is not a narrowed gate. The two opt-in experiment surfaces remain separately qualified by the driver. Two workers will be slower than the standard pool and may expose a timeout/capacity limit; inspect the real plan/timings on Blaxel before the gate. If the named private capacity cannot complete it within the available runner limits, report that blocker to the supervisor rather than silently use the shared pool, omit shards or change retry budgets. + +This private full-gate arrangement replaces the proposal's earlier shared-`poc` final-run plan. Retrieve all evidence before deleting private hosts, and report cleanup that cannot be confirmed. Retain working documents while the PR is draft; perform pre-merge docs preparation only after implementation and integrated validation are complete. Josh's spec acceptance and the repository's required GitHub checks remain merge obligations; neither this plan nor an independent PASS claims they have happened. + +## Current limits + +No new tests or implementation exist at this checkpoint and no execution has occurred. The coordinator's local preflight capabilities, credentials and capacity; actual provider behavior during renewal, label changes and interrupted dispatch; browser temporary placement; and private full-gate duration remain unmeasured. The plan carries those checks into the slices that can establish them. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. diff --git a/docs/working/test-harness-resource-cleanup-proof-review-r2.md b/docs/working/test-harness-resource-cleanup-proof-review-r2.md new file mode 100644 index 00000000..ef676d09 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-proof-review-r2.md @@ -0,0 +1,61 @@ +# Proof review, round 2: test-harness resource cleanup (PR #12, 3fa15d47) + +**Verdict: PASS.** The placement spec delta gives the live tests and their deliberately failing fixtures a home that fits the registry and runner as they are, with no new kind or suite, and it states Josh's host rule outright. The five proof refinements are applied, one of them better than I asked. What remains is the proof binding the author deliberately held back until this spec review: naming the two surfaces, the coordinator host and the two selection commands in the live declarations. That is a bounded follow-up derived from a paragraph reviewed here, and I do not think it needs another independent round if it stays within what this report describes. Three small refinements follow. + +- Reviewed: `git diff 9fd51f49 3fa15d47` for specs, proofs and working documents; the whole current proof delta against `origin/main` for the Blaxel, reporting, testing-policy, themes, artifacts and bridge proofs; the placement delta against the registry, runner and Blaxel specs and the runner's selection code; the reviewer checklist and the proof policy. +- Branch state: contains `origin/main` (`a8e73fbf`); the copied round 1 review is identical to mine; the spec and proof indexes match the files. +- I ran nothing. Every check is a read of files in the checkout or of git history. + +## The placement spec delta + +| Claim | Where | Checked | +| --- | --- | --- | +| Two `kind: "experiment"` surfaces: `experiment:blaxel-harness` (driver) and `experiment:blaxel-harness-fixtures` (native fixtures, tag `blaxel-harness-fixture`) | `test-registry.md:97` `^blaxel-harness-surfaces` | Uses the existing kind; the sentence that counted the experiment suite's "two placement classes" is removed, so nothing now miscounts. `all` and `e2e-ci` select by kind (`:90-93`), so both surfaces are excluded from broad runs and ordinary CI without a new rule. | +| The driver runs as `npm test -- local --surface experiment:blaxel-harness` on a dedicated credentialed Blaxel coordinator sandbox outside the worker pool, from a committed and pushed checkout, never on a shared developer host or a leased worker | `test-runner.md:147-149` `^blaxel-harness-placement` | States Josh's rule as spec. The local provider accepts every registered suite and surface (`:70`). | +| `--tag blaxel-harness-fixture` dispatches a planned `--suite selection --surfaces …` run; `--surface experiment:blaxel-harness-fixtures` dispatches a targeted run; workers read the committed registry, no forwarded hook | `:151` | Matches the runner today: `--surface/--file/--grep` take the targeted path and `--package/--runner/--tag` become `--suite selection --surfaces` (`scripts/test/cli.mjs:491-510`). The coordinator already accepts `--suite selection` with `--surfaces` (`blaxel-testshards.md:117-118`). A two-shard planned run of one surface needs the existing `--shards 2 --allow-shard-count-override 1`. | +| Deliberate fixture failures stay failed native results; the driver asserts expected outcomes and cleans up its coordinators and pool, reporting what it could not clean | `:151` | Same precedent as the `isolated-github-only` fixtures: registered surfaces that are never in `all`. | +| Blaxel's `Testing` section points at the placement | `blaxel-testshards.md:172` | Reference only. | + +The delta is proportionate: four sentences in the registry, two paragraphs in the runner, one reference. It changes no lock, recovery, cleanup or diagnostic promise. + +## The proof refinements + +1. **Citation grades and the proxy.** Every assertion whose walk is being moved to the built application now carries the mixed form, "policy-grade test for the existing walk: …; built-application boundary: test to be written", and names the stable-front proxy as harness transport: the product theme assertions (`^theme-ac-selection`, `^theme-ac-executable-browser`, `^theme-ac-sandboxed-frames`, `^theme-ac-frame-appearance`, `^theme-ac-active-package`, `^theme-ac-appearance`), the delivery assertions (`^theme-delivery-t-active-package-refresh`, `^theme-delivery-t-frame-browser`, `^theme-delivery-t-frame-appearance-browser`), the demo-mode walks and the bridge's pointer and demo-inert assertions. The author found the frame, pointer and bridge assertions I had not listed; the reset walk and the already-built walks keep plain coverage, correctly. +2. **The guard seam** is now an ordinary repository test on Linux, needing no credentials (`proofs/arch/test-runner/blaxel-testshards.md:78`). +3. **The reporting hook** says it serves the local parent/child seam only and is not forwarded to workers (`reporting.md:22`). The worker-side selection commands are now in the spec and await the binding below. +4. **Launch cleanup** is folded into the real worker seam (`testing-policy.md:25`) and the standalone assertion is gone. The response is right that my premise was too narrow: the policy's cleanup section does say that cleanup of long-running modules is core to correctness, below its motion subsection. The consolidation still stands on its own merits, one fixture and one launch sequence fewer. +5. **Upload paths and the `temporary-cleanup` phase** are observed inside `^blaxel-t-retrieval` on the run that already happens. + +## What is left: the binding + +The Blaxel proof's coverage model still says the live tests are "an explicit paid-service selection" (`:15`), and the shared live declaration (`:84`) and `^reporting-t-resource-live` still name no surface, host or selection. The response records this as deliberately pending spec convergence and lists what it will add. For that follow-up to stay bounded, it should contain only: + +- in the shared live declaration: the driver surface, the fixture surface, the coordinator host rule, and the two selection commands, with links to `^blaxel-harness-placement` and `^blaxel-harness-surfaces`; +- the coverage-model sentence replaced by one that says where the live tests live and that they are excluded from `all` and ordinary CI by kind; +- one contract in the registry or runner proof over the committed registry: `all` and `e2e-ci` resolve without either surface, the tag resolves to the fixture surface alone, and the driver's dispatched selections name only the fixture surface. That is a source-level contract, not a live run. + +Within that scope I would not ask for another round. Anything beyond it, the owner judges under the bounded-refinement rule. + +## Refinements (none blocking) + +1. **The runner's `Selectors` paragraph should say what the new section relies on.** `test-runner.md:70` says narrower remote work uses "surface, file, or grep selection through the target path" and is silent on `--package`, `--runner` and `--tag`, which the code sends to the planned selection path. The placement paragraph depends on that path. One clause at `:70` makes the spec self-consistent instead of leaving the only statement of the planned-selection route inside a specialized section. +2. **`test-runner.md:92` now has a third opt-in placement it does not mention.** "The `telemetry-posthog-roundtrip` and `daemon-acceptance` suites are not broad verification" could add "and the Blaxel harness surfaces ([#Blaxel harness qualification](#Blaxel harness qualification))" so a reader of the guardrail section sees all three. +3. **Minimum shard shape for the planned fixture run.** The fixture surface must hold at least two files for `^blaxel-t-retrieval`'s two-shard run; worth one clause in the fixture surface's description so the first implementer does not discover it from a plan failure. + +## Reviewer checklist + +- **Authority and placement.** Surface declarations in the registry spec, execution placement in the runner spec, a reference from the provider spec. Nothing spec-shaped remains in the proofs once the binding lands. +- **Proof coverage, mocking and boundaries.** Unchanged from round 1 and still sound; the consolidated worker seam keeps its real crossing. +- **Citation grades.** Honest everywhere I looked. +- **References.** All new anchors resolve; indexes regenerated. +- **Size and simplicity.** No new kind, suite, hook or framework; the driver reuses canonical commands. + +## Evidence and limits + +- Read the full spec and proof diffs, the response's round 1 section, and the current Blaxel, reporting and testing-policy proofs; checked the placement claims against `cli.mjs` and the coordinator option table; confirmed the policy's cleanup sentence the response cites. +- Not established: whether a `local` run of the driver surface on the coordinator sandbox passes local preflight's capability checks as that sandbox is provisioned, which the first run will show. + +## Housekeeping + +- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel. `git fetch origin` updated remote-tracking refs. +- Primitive produced no output during this review, so I have nothing to report on its usefulness. The response records a Primitive decision (`dec_7893f958`) from the author's session; I have not seen it and do not assess it. diff --git a/docs/working/test-harness-resource-cleanup-proposal.md b/docs/working/test-harness-resource-cleanup-proposal.md index 6df5420a..426d8471 100644 --- a/docs/working/test-harness-resource-cleanup-proposal.md +++ b/docs/working/test-harness-resource-cleanup-proposal.md @@ -211,7 +211,7 @@ After proposal convergence, derive the smallest changes in the existing owners: - [`blaxel-testshards.md`](../../specs/arch/test-runner/blaxel-testshards.md): exclusive reservation through setup, process teardown and result retrieval; renewal, automatic recovery after lost renewal, owned-process cleanup before reuse, and the explicit abandoned-report exception; actual older-runner behavior; pool-script locking; attempt temporary directories, recurring exact-prefix cleanup and bounded worker retention. Replace its blind TTL escape and shell-exit unlock expectations rather than leaving conflicting requirements. - [`reporting.md`](../../specs/arch/test-runner/reporting.md): minimum resource observations and artifact references, bounded content, unavailable evidence, retention alongside failed/recovered native attempts, and recovery receipts that disclose discarded orphan reports. -Use existing lifecycle and reporting contracts in `test-runner.md` and `sharded-execution.md`, adding the cross-references or ownership corrections required by the outer Blaxel lock. The proof review also exposed missing authority for the automated live driver and its deliberate-failure fixtures: the proposed `test-registry.md` and `test-runner.md` delta places them in separately selected experiment surfaces, with the driver on a dedicated credentialed Blaxel coordinator and the fixtures on its private pool. This keeps repeatable live coverage without adding a kind or suite. That placement delta awaits independent spec review before dependent proof derivation. Existing isolated-GitHub-only lifecycle fixtures keep their placement. No `preflight.md` delta is planned. Optional investigation instrumentation remains optional. +Use existing lifecycle and reporting contracts in `test-runner.md` and `sharded-execution.md`, adding the cross-references or ownership corrections required by the outer Blaxel lock. The proof review also exposed missing authority for the automated live driver and its deliberate-failure fixtures: the `test-registry.md` and `test-runner.md` delta places them in separately selected experiment surfaces, with the driver on a dedicated credentialed Blaxel coordinator and the fixtures on its private pool. This keeps repeatable live coverage without adding a kind or suite. Proof review round 2 approved that placement delta; its dependent proof binding is complete. Existing isolated-GitHub-only lifecycle fixtures keep their placement. No `preflight.md` delta is planned. Optional investigation instrumentation remains optional. Update the mirrored proofs after independent spec convergence. Test setup changes may also require proof-only corrections to descriptions of the affected browser walks; they do not change their product promises. Do not prescribe a new framework or adopt unrelated harness internals into spec authority. @@ -237,9 +237,9 @@ For the final gate, provision a separate sandbox named **`tv-cleanup-coordinator Supply the coordinator's credentials through the Blaxel SDK's `process.exec` environment: `BL_WORKSPACE` and an authorized `BL_API_KEY` for its outbound Blaxel calls, plus `BLAXEL_TV_GH_TOKEN` for repository access. The launching process reads the existing authorized secrets into memory and passes them directly in that environment object; do not interpolate their values into shell commands, uploaded scripts, output, or reports. The inspected SDK accepts the Blaxel pair without a CLI login/config file, and `readBlaxelGithubToken()` accepts `BLAXEL_TV_GH_TOKEN` when no `.blaxel-gh-token` file is present. Use that environment path in the fresh coordinator, with `GIT_ASKPASS` reading the environment for its initial fetch; the existing runner passes only the GitHub token onward to workers as `GH_TOKEN`. Do not copy local credential files or persist tokens in the coordinator's filesystem, sandbox configuration, or Git configuration. Confirm that the required credential is available and valid for the run before dispatch; its availability is not established by this proposal. End the credential-bearing process after downloading its reports and dispose the dedicated coordinator. This describes ephemeral process injection, not a claim about provider-internal retention. -The experiment plan adds at most three sandboxes. The spec's pool-size and tier examples are not a current capacity inventory: inspect actual workspace usage and provider limits before provisioning/dispatch. Do not claim slots are reserved or alter shared workers to obtain them. Delete this contribution's experiment workers through the pool script after retrieving their reports and before final standard 36-shard verify, leaving only its coordinator. That final run uses the normal `poc` acquisition path without intentional contamination or interruption; it does not require a pool-wide wait for other runs or a coordinated update of other branches. Archive its reports before disposing the coordinator. Record the exact tested revision, failures, recovered flakes, and any measurement limits. Ready-PR CI remains the repository's merge requirement; the contribution's execution evidence stays on Blaxel as instructed. +The experiment plan adds at most three sandboxes. The spec's pool-size and tier examples are not a current capacity inventory: inspect actual workspace usage and provider limits before provisioning/dispatch. Do not claim slots are reserved or alter shared workers to obtain them. The [implementation plan](test-harness-resource-cleanup-plan.md#Integrated result and final validation) keeps all validation, including complete verification, on the named private hosts. After the live pool-deletion acceptance, provision fresh private workers and use the supported two-shard override for every `all` index. This preserves full selection while running more slowly than the standard pool. If the private capacity or available timeout blocks completion, report it rather than move validation to shared workers or omit coverage. Retrieve reports before deleting the private workers and coordinator. Record the exact tested revision, failures, recovered flakes, and any measurement limits. Ready-PR CI remains the repository's merge requirement; the contribution's execution evidence stays on Blaxel as instructed. -Keep browser-version alignment, shard count, browser request limits, retries, and production memory behavior outside this contribution. Forced GC is an investigation tool, not a proposed mitigation. Preserve main's demo-mode teardown fix; the drag flake has no established resource-exhaustion cause. +Keep browser-version alignment, the default shard count, browser request limits, retry defaults, and production memory behavior outside this contribution. Forced GC is an investigation tool, not a proposed mitigation. Preserve main's demo-mode teardown fix; the drag flake has no established resource-exhaustion cause. ## Review status @@ -249,4 +249,4 @@ Proposal round 2 at `d3570bf4`: FAIL because dead-coordinator lock recovery is u Proposal round 3 at `57a701d0`: **PASS**, relayed by the supervisor. The [review](test-harness-resource-cleanup-review-r3.md) is preserved unchanged. Its bounded refinements are applied here: fixed recovery interval, deletion of abandoned workers despite failed process cleanup, ordinary skipping of live legacy work, and trimmed receipt/provisioning language. The [response](test-harness-resource-cleanup-review-response.md#Round 3: convergence and spec derivation) records the judgments. The proposal and spec deltas have converged: spec round 1 at `cd8aa453` also passed. Its bounded refinements and the derived proof checkpoint are recorded in the response; independent proof review is next. -Proof round 1 at `9fd51f49`: **FAIL** on missing spec-owned placement for live qualification and deliberate-failure fixtures. The [review](test-harness-resource-cleanup-proof-review-r1.md) is preserved unchanged. The [response](test-harness-resource-cleanup-review-response.md#Proof round 1: placement finding and refinements) proposes the narrow placement delta for independent spec review and applies the independent proof refinements. Dependent live-proof binding follows that review; test and implementation work has not begun. No tests, builds, remote experiments, or worker operations were performed for this checkpoint. Unknowns are live provider behavior during failed dispatch/recovery, today's worker browser versions, actual `TMPDIR` placement and post-mitigation memory savings, current capacity/credential availability, and observed recovery timing. Recovery waits at most 20 minutes after the last renewal by default, then acquisition and bounded cleanup; failed cleanup prevents test reuse and explicit pool deletion remains available. Josh's spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. +Proof round 1 at `9fd51f49`: **FAIL** on missing spec-owned placement for live qualification and deliberate-failure fixtures. The [review](test-harness-resource-cleanup-proof-review-r1.md) and [response](test-harness-resource-cleanup-review-response.md#Proof round 1: placement finding and refinements) retain that history. Proof round 2 at `3fa15d47`: **PASS**, including the placement delta. Its [report](test-harness-resource-cleanup-proof-review-r2.md) permits the bounded proof binding now applied. The [implementation plan](test-harness-resource-cleanup-plan.md) is ready for independent review; tests and implementation have not begun. No tests, builds, remote experiments, or worker operations were performed for this checkpoint. Unknowns are live provider behavior during failed dispatch/recovery, today's worker browser versions, actual `TMPDIR` placement and post-mitigation memory savings, current capacity/credential availability, private full-gate duration, and observed recovery timing. Recovery waits at most 20 minutes after the last renewal by default, then acquisition and bounded cleanup; failed cleanup prevents test reuse and explicit pool deletion remains available. Josh's spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 8b249ef3..3ed611f8 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -1,6 +1,6 @@ # Resource-cleanup proposal: review response -Proposal round 3 at `57a701d0` and spec round 1 at `cd8aa453` both **PASS**, relayed by the supervisor. Proof round 1 at `9fd51f49` **FAILS** on missing spec-owned placement for the live tests and their deliberate-failure fixtures. The [proof response below](#Proof round 1: placement finding and refinements) returns that narrow delta to spec review and records the independent proof corrections. Earlier sections retain the response history. Reviewer: Claude Fable (Nina), arranged by the supervisor. +Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, and proof round 2 at `3fa15d47` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. The [round 2 response](#Proof round 2: convergence and implementation planning) records the bounded bindings/refinements and the three-slice plan ready for independent review. Earlier sections retain the response history. Reviewer: Claude Fable (Nina), arranged by the supervisor. Independent round 1 reviewed `1652787e`. The [full review](test-harness-resource-cleanup-review-r1.md) remains unchanged: FAIL on two blockers. The response committed at `99b7ed93` narrowed the work pending Josh's decisions. The following sections through round 2 record that history, not the current review verdict. @@ -153,3 +153,21 @@ After placement spec convergence, complete the dependent proof binding: name bot Validation is document/source inspection, exact comparison of the saved review, Markdown index regeneration and Git diff/whitespace inspection. No tests, builds, browser/server launches, provider experiments or worker operations ran. Test and harness implementation remains unstarted. Primitive's file check returned existing recovery and placement decisions with `truncated: true`; it was not a complete constraint check. No decision was overridden or reconciled. The latest 20 decisions contained no duplicate of the proposed automated experiment placement; it is recorded as agent-attributed `dec_7893f958`, with the repeatability rationale and operator-only alternative. That record does not substitute for independent spec convergence or Josh's PR review. + +## Proof round 2: convergence and implementation planning + +The supervisor relayed **PASS** on `3fa15d47`; the [complete report](test-harness-resource-cleanup-proof-review-r2.md) is preserved verbatim. The reviewer approved the placement spec delta and explicitly allowed the deferred proof binding as a bounded follow-up without another round. Complete that binding: both experiment surfaces, the dedicated coordinator host, planned/tag and targeted/surface commands, and one source-level registry/driver selection contract. Reporting's live assertion shares the same driver and placement. No new live scenario, suite, kind or test hook is introduced. + +| Refinement | Assessment and disposition | +| --- | --- | +| 1: general selector description | Accepted. State package/runner/tag planned selection alongside the existing target path; this documents the current routing used by the reviewed placement. | +| 2: broad-run guardrail description | Accepted. Cross-reference the Blaxel harness surfaces and their exclusion from `all` alongside the other opt-in suites. | +| 3: minimum fixture shape | Accepted in the shared proof declaration and implementation plan. At least two native files are needed for the reviewed two-shard, file-atomic fixture run. This is derived test design, so it does not need a permanent minimum inventory rule in the registry spec. | + +The work is not clearly one slice. The [implementation plan](test-harness-resource-cleanup-plan.md) proposes three coherent slices: worker-owned builds plus the five built-app conversions; bounded diagnostic capture/reporting; then exclusive leases, recovery, scratch/pool operations and the shared live qualification. Keeping the entire remote ownership lifecycle together avoids partially integrated exclusion rules. Separating diagnostics and helpers makes their real framework/process boundaries reviewable without repeating the paid-service scenarios. Slice 3 explicitly owns the pending remote diagnostic evidence and killed-worker backstop from the earlier slices. Each slice has its red/green criteria and expected baseline; the integrated result has a separate review and full gate. + +Every test/build runs on a pushed revision on Blaxel. All worker-affecting validation uses `rose-cleanup`; the driver runs on `tv-cleanup-coordinator-rose` outside the pool. The plan also chooses the same private two-worker pool for complete `all` verification using the existing shard-count override, replacing the proposal's earlier shared-pool final-run arrangement. The complete selection still qualifies under the runner spec; private capacity and timeout headroom are unmeasured and must be reported if they block the gate. No silent shared-pool fallback is authorized by this plan. + +Validation for this planning checkpoint is document/source inspection, exact review-copy comparison, Markdown index regeneration, and Git diff/whitespace inspection. No tests, builds, browser/server launches, provider experiments or worker operations ran. Implementation waits for the supervisor-arranged independent plan review. Josh's spec review remains at PR time. + +Primitive surfaced the already-recorded opt-in experiment choice after the prior turn. That was consistent context but added no new requirement. Its edit hook reported a truncated governing-set scan, so it did not establish a complete constraint check; no decision was overridden. This turn follows the settled direction; the slice ordering and private validation arrangement are contribution-scoped execution choices, so no new durable decision was recorded. diff --git a/proofs/arch/test-runner/blaxel-testshards.md b/proofs/arch/test-runner/blaxel-testshards.md index 1a1b1ac2..13788a80 100644 --- a/proofs/arch/test-runner/blaxel-testshards.md +++ b/proofs/arch/test-runner/blaxel-testshards.md @@ -12,7 +12,7 @@ Lease decisions and failure ordering are checked cheaply at the worker/coordinat The live acceptances below enter through the canonical CLI on a dedicated pool: normal retrieval, refusal while another coordinator holds the worker, recovery after coordinator death, and pool creation/deletion. They share a small committed fixture selection and fresh workers rather than repeating a full suite for every failure permutation. Both planned and targeted execution paths are crossed. Resource retention composes with [reporting's diagnostics assertions](reporting.md#Resource diagnostics); helper teardown belongs to [the testing-policy proof](../testing-policy.md#Worker-owned builds). -The spec's [Testing](../../../specs/arch/test-runner/blaxel-testshards.md#Testing) directives require real API/transport, fresh provisioning, and observed Node/Chromium temporary paths. Private workers and bounded disposable processes supply those observations; no deliberately exhausted host or shared-pool contamination is needed. The live tests are an explicit paid-service selection, not a recursive addition to every broad run. Mocks below forfeit those crossings to the named live assertions. New assertions are obligations, not evidence of runs already performed. +The spec's [Testing](../../../specs/arch/test-runner/blaxel-testshards.md#Testing) directives require real API/transport, fresh provisioning, and observed Node/Chromium temporary paths. Private workers and bounded disposable processes supply those observations; no deliberately exhausted host or shared-pool contamination is needed. The live driver and its native fixtures live in `experiment:blaxel-harness` and `experiment:blaxel-harness-fixtures`, excluded from `all` and ordinary CI by their experiment kind. Mocks below forfeit those crossings to the named live assertions. New assertions are obligations, not evidence of runs already performed. `test/repo/blaxel-stale-owner-audit.test.ts` checks diagnostic construction and coordinator ordering on every host. Its live diagnostic cases require Linux procfs and run only on Linux; they establish no macOS backend coverage. `test/repo/test-runner-provider.test.ts` exercises shell activation ordering with an authored nvm replacement in a shell isolated from host startup files. That contract check does not install nvm or establish real worker activation. @@ -81,7 +81,7 @@ The remaining coverage lives in `test/repo/test-runner-provider.test.ts`. - **Contract — cleanup uncertainty.** The cleaner's decision layer consumes authored filesystem-probe results and removal errors. An unexpected mount boundary is skipped, and an incomplete removal is reported without claiming the whole directory was reclaimed; neither permits new test dispatch under [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#Temporary files). These mocked conditions forfeit actual filesystem operations to the cleanup seam and live retrieval assertion; a real mount or full disk adds no necessary coverage. (test to be written) ^blaxel-t-scratch-errors -Shared live declaration: the following acceptances spawn the canonical CLI with real argv and inspect its exit and retained run directory. A fresh dedicated pool, committed native fixture tests, known leftover directories and unrelated sentinels are fixtures. The coordinator, SDK, worker scripts, native runners, transport, ownership audits and filesystem operations are real. The report-download rendezvous above is the only scheduling hook; no mechanism is mocked. A bounded test driver owns and finally cleans up its coordinators and private workers. These tests do not invoke fixtures restricted to isolated GitHub hosts. +Shared live declaration: the following acceptances run in `experiment:blaxel-harness` via `npm test -- local --surface experiment:blaxel-harness` on a dedicated credentialed Blaxel coordinator sandbox outside the worker pool, from a committed and pushed checkout, never a shared developer host or a leased worker, under [harness placement](../../../specs/arch/test-runner/test-runner.md#^blaxel-harness-placement). They spawn the canonical CLI with real argv and inspect its exit and retained run directory. A fresh dedicated pool, at least two committed native fixture files in `experiment:blaxel-harness-fixtures`, known leftover directories and unrelated sentinels are fixtures. The two files permit the planned run's file-atomic two-shard assignment. Under [surface selection](../../../specs/arch/test-runner/test-registry.md#^blaxel-harness-surfaces), `npm test -- blaxel --tag blaxel-harness-fixture --pool --shards 2 --allow-shard-count-override 1` delegates `--suite selection --surfaces experiment:blaxel-harness-fixtures`; `npm test -- blaxel --surface experiment:blaxel-harness-fixtures --pool ` takes the targeted path. Workers read the committed registry. The coordinator, SDK, worker scripts, native runners, transport, ownership audits and filesystem operations are real. The report-download rendezvous above is the only scheduling hook; no mechanism is mocked. A bounded test driver owns and finally cleans up its coordinators and private workers. These tests do not invoke fixtures restricted to isolated GitHub hosts. - **Acceptance — reports before reuse.** A small planned run with at least two shards traverses real acquisition, architecture verification, checkout, runtime/dependency setup, execution and log/report transport under [the shard lifecycle](../../../specs/arch/test-runner/blaxel-testshards.md#Shard lifecycle). Each uploaded script path lies inside its current shard attempt directory. The retained setup phases include `temporary-cleanup`, with cleanup counts and free-space evidence. While it waits at the download rendezvous, observed renewals keep the leases valid; a competing canonical run cannot acquire the occupied capacity or alter its result files. After retrieval, the original native reports, setup/cleanup evidence and diagnostics have the original identities and readable local paths, scratch is gone, and the workers become available to a targeted run. Legacy-prefix and stale-scratch fixtures disappear, unrelated sentinels and persistent caches survive, and the next run does not accumulate an older results tree. The targeted run crosses its distinct dispatch path too. This proves [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release) and [scratch lifetime](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-scratch); the shared diagnostics observations are owned by [reporting](reporting.md#^reporting-t-resource-live). (test to be written) ^blaxel-t-retrieval diff --git a/proofs/arch/test-runner/reporting.md b/proofs/arch/test-runner/reporting.md index 53c36714..858e212c 100644 --- a/proofs/arch/test-runner/reporting.md +++ b/proofs/arch/test-runner/reporting.md @@ -54,4 +54,4 @@ The event-normalization assertions are **contract tests** over recorded native-r - **Contract — retained links and attempt history.** Provider normalization and infrastructure-attempt aggregation consume authored shard/task envelopes referring to real temporary sidecar files, with recorded native failed/recovered fixtures. Under [normalized results](../../../specs/arch/test-runner/reporting.md#Normalized surface result) and [resource diagnostics](../../../specs/arch/test-runner/reporting.md#Resource diagnostics), each advertised `resourceDiagnosticsPaths` entry exists relative to the final run directory; earlier attempt evidence survives a later successful infrastructure attempt. Missing native output remains incomplete and cannot manufacture a selected-surface result. Samples and kernel excerpts appear in neither compact console output nor timing events. Mocked provider envelopes forfeit remote creation/download to the live assertion; recorded native inputs forfeit attempt capture to the real parent seam and existing native-recorder assertions above. (test to be written) ^reporting-t-resource-links -- **Acceptance — evidence reaches the caller.** Share the dedicated-pool canonical CLI runs in [the provider retrieval acceptance](blaxel-testshards.md#^blaxel-t-retrieval), with a committed fail-once native fixture that recovers under the default retry budget and a fixture that fails all attempts. Planned and targeted runs retain the native outcomes and their start/end sidecars after worker scratch is removed; every normalized diagnostic link opens a downloaded file with the correct run/shard/surface identity and native-attempt references. The failed run still fails, the recovered case retains both attempts, and any inaccessible kernel evidence says so. The provider's [crash acceptance](blaxel-testshards.md#^blaxel-t-recovery-live) supplies the downloaded recovery receipt and its disclosed report loss. Boundary: real CLI, Blaxel API, native runners and report files, with the provider proof's scheduling hook and no mocks. This proves [resource retention](../../../specs/arch/test-runner/reporting.md#Resource diagnostics) end to end without a resource-exhaustion experiment. (test to be written) ^reporting-t-resource-live +- **Acceptance — evidence reaches the caller.** Share the dedicated-pool canonical CLI runs in [the provider retrieval acceptance](blaxel-testshards.md#^blaxel-t-retrieval), under its shared `experiment:blaxel-harness` declaration and dedicated coordinator placement. Its `experiment:blaxel-harness-fixtures` selection includes a committed fail-once native fixture that recovers under the default retry budget and a fixture that fails all attempts. Planned and targeted runs retain the native outcomes and their start/end sidecars after worker scratch is removed; every normalized diagnostic link opens a downloaded file with the correct run/shard/surface identity and native-attempt references. The failed run still fails, the recovered case retains both attempts, and any inaccessible kernel evidence says so. The provider's [crash acceptance](blaxel-testshards.md#^blaxel-t-recovery-live) supplies the downloaded recovery receipt and its disclosed report loss. Boundary: real CLI, Blaxel API, native runners and report files, with the provider proof's scheduling hook and no mocks. This proves [resource retention](../../../specs/arch/test-runner/reporting.md#Resource diagnostics) end to end without a resource-exhaustion experiment. (test to be written) ^reporting-t-resource-live diff --git a/proofs/arch/test-runner/test-registry.md b/proofs/arch/test-runner/test-registry.md index ff3af3aa..2675d3d7 100644 --- a/proofs/arch/test-runner/test-registry.md +++ b/proofs/arch/test-runner/test-registry.md @@ -32,7 +32,7 @@ These run in-process as pure functions over the loaded registry, plus real files - `validateRegistry` ignores a runner config placed outside the canonical include roots (e.g. under `prototypes/` or `experiments/`). - The `all` suite resolves to the unit and non-agent e2e execution groups, in `order`, includes `unit:build-config`, and the `unit:workspaces` group contains the package unit surfaces in declared order. - Every lifecycle fault surface retains `kind: "experiment"` and `tag:isolated-github-only`; none is selected by `all`. +- **Contract — live harness selection.** Load the committed registry with the real selector and inspect the committed driver's dispatch selections. Both [Blaxel harness surfaces](../../../specs/arch/test-runner/test-registry.md#^blaxel-harness-surfaces) have `kind: "experiment"`; `all` and `e2e-ci` select neither, and tag `blaxel-harness-fixture` selects only `experiment:blaxel-harness-fixtures`. The driver's planned and targeted selections name only that fixture surface, never the driver, as required by [harness placement](../../../specs/arch/test-runner/test-runner.md#^blaxel-harness-placement). This is a source-level contract with no mocks or provider calls; real dispatch is carried by [the provider retrieval acceptance](blaxel-testshards.md#^blaxel-t-retrieval). (test to be written) ^registry-t-blaxel-harness-selection - The `unit:build-config` surface owns `test/repo/build-config-integrity.test.ts`; `unit:root` excludes that file. - The `telemetry-posthog-roundtrip` suite resolves to the `telemetry-posthog-roundtrip:integration` surface, and that surface owns `packages/server/test/telemetry-posthog.integration.test.ts`; `unit:server` excludes that file. - The `daemon-acceptance` suite resolves to `daemon-acceptance:cli`, and that surface exclusively owns `test/node/daemon-acceptance.test.ts`; `e2e:node` excludes that file. The surface declares the `daemon-test-host` preflight and is absent from `all`. - diff --git a/specs/arch/test-runner/test-runner.md b/specs/arch/test-runner/test-runner.md index 15e4e252..b1c3b9bc 100644 --- a/specs/arch/test-runner/test-runner.md +++ b/specs/arch/test-runner/test-runner.md @@ -67,7 +67,7 @@ A *selector* chooses which tests run; the runner resolves selectors against [tes | `--runner ` | Surfaces using a runner. | | `--tag ` | Surfaces carrying a registry tag, such as `browser` or `electron`. | -The local provider accepts every registered suite. A Blaxel suite selection accepts only `all`, `unit`, or `e2e`; narrower remote work uses surface, file, or grep selection through the target path. +The local provider accepts every registered suite. A Blaxel suite selection accepts only `all`, `unit`, or `e2e`; narrower remote work uses surface, file, or grep selection through the target path, or package, runner, or tag selection through the planned selection path. `--file` must resolve to exactly one owning surface: if no surface owns the file the runner fails; if several do, it fails and asks for `--surface` to disambiguate. On the local execution path, a `--file` or `--grep` selection against a surface that does not declare support for it is rejected while building that surface's command. The targeted Blaxel path does not currently apply this `supports` check. The runner parses these selectors itself and rejects unknown positional arguments; the sole positional accepted is the `verify` provider. @@ -89,7 +89,7 @@ Production attestations have exactly two source classes: a publication-qualifyin A *broad run* is a provider shortcut that runs the full `all` set with no narrowing selector. A broad run skips lint and type-check, so the provider shortcuts refuse it with a message directing the caller to `npm run verify`. -The `telemetry-posthog-roundtrip` and `daemon-acceptance` suites are not broad verification. They are specialized, opt-in local suites and are excluded from `all`. The first crosses the live telemetry ↔ PostHog boundary; the second replaces the active global `tv` package and the literal `com.television.server` user service on a designated developer host. Build-config-integrity tests are not part of either suite: `unit:build-config` is a normal unit surface in `all` and uses no secrets, network, or host service mutation. +The `telemetry-posthog-roundtrip` and `daemon-acceptance` suites and [Blaxel harness surfaces](#Blaxel harness qualification) are not broad verification and are excluded from `all`. The two suites are specialized, opt-in local suites: the first crosses the live telemetry ↔ PostHog boundary; the second replaces the active global `tv` package and the literal `com.television.server` user service on a designated developer host. The harness surfaces have the separate coordinator/worker placement below. Build-config-integrity tests are not part of either suite: `unit:build-config` is a normal unit surface in `all` and uses no secrets, network, or host service mutation. The guardrail trips only on the effective `all` suite with nothing narrowing it: a bare provider (which defaults to `--suite all`), an explicit `--suite all` or `--all`, or a bare `--grep` with no other selector. Everything else passes through: From 04a6b275317e81af9141e6c1ba1594a74dde5863 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:35:02 +0000 Subject: [PATCH 09/64] Correct resource cleanup validation plan after review --- ...harness-resource-cleanup-plan-review-r1.md | 61 +++++++++++++++++++ .../test-harness-resource-cleanup-plan.md | 24 +++++--- .../test-harness-resource-cleanup-proposal.md | 6 +- ...arness-resource-cleanup-review-response.md | 21 ++++++- 4 files changed, 98 insertions(+), 14 deletions(-) create mode 100644 docs/working/test-harness-resource-cleanup-plan-review-r1.md diff --git a/docs/working/test-harness-resource-cleanup-plan-review-r1.md b/docs/working/test-harness-resource-cleanup-plan-review-r1.md new file mode 100644 index 00000000..fa346984 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-plan-review-r1.md @@ -0,0 +1,61 @@ +# Plan review, round 1: test-harness resource cleanup (PR #12, 70f27ccb) + +**Verdict: FAIL, on one item.** The three slices are coherent, every one of the 31 pending proof obligations has a slice, the red/green criteria and intermediate baselines are honest, the scope stays inside the converged specs, and nothing in the plan runs on the shared development host. The blocker is the final gate. The plan moves full verification from the shared `poc` pool, where the converged proposal put it, to the two private workers with `--shards 2`. The repository's own timing baseline says each of those two shards would carry about 441 seconds of measured test compute before any setup, against the 480-second worker timeout the canonical runner cannot raise, on a tier that cannot hold a larger private pool. The gate as planned will not complete, and the plan's contingency is to stop and report at the end of the work rather than decide now. Returning the gate to `poc` at the standard shard count, run from the dedicated coordinator sandbox, clears it in a sentence. Four refinements follow, none blocking. + +- Reviewed: `docs/working/test-harness-resource-cleanup-plan.md` at `70f27ccb` against the workflow's Planning and slices section, the testing policy's iteration and verification rules, the runner spec's publication-qualification and verify rules, the reviewer checklist, the proof obligations at this revision, the runner's option forwarding, and the committed timing baseline. Also the bounded proof binding and two spec clauses since `3fa15d47`. +- Branch state: contains `origin/main` (`a8e73fbf`); the copied round 2 proof review is identical to mine. +- I ran nothing. I read `test/timing-baseline.json` with a small script that sums its recorded medians; nothing was built, tested or changed. + +## The binding since round 2 stayed in scope + +The shared live declaration now names both surfaces, the coordinator host rule and both selection commands, the coverage model names the placement, and the registry proof carries one source-level selection contract (`^registry-t-blaxel-harness-selection`). The runner spec gained the `Selectors` clause and the guardrail cross-reference I asked for. Nothing else moved. The one unrelated pending anchor in the tree (`proofs/arch/cli/startup-bind-failure.md`) predates this branch. + +## Slices, obligations, baselines + +| Slice | Obligations placed | Baseline stated | +| --- | --- | --- | +| 1: worker-owned builds and the five built-application walks | `^testing-t-build-lifetime`, `^testing-t-build-worker-seam`; the eleven theme, delivery, artifact and bridge citations graded "built-application boundary: test to be written" | Green for its obligations; killed-worker backstop assigned to slice 3 (`plan.md:37`). | +| 2: bounded diagnostics and report transport | `^reporting-t-resource-bounds`, `-parent`, `-links` | Explicit: `^reporting-t-resource-live` pending, lease/download race still present, checkpoint not safe for concurrent reuse (`:51`). | +| 3: exclusive lifecycle, scratch, pool mutation, live qualification | `^blaxel-t-lease-decisions`, `-lease-order`, `-worker-guard`, `-scratch-cleanup`, `-scratch-errors`, `-pool-order`, the `^blaxel-pool-pagination` extension, `^registry-t-blaxel-harness-selection`, and the five live assertions including `^reporting-t-resource-live` | Resolves every earlier baseline (`:74`). | + +The slice judgment is sound: helper ownership and shipped-asset loading have no dependence on the coordinator; diagnostics have their own native-runner boundary; and the lease, dispatch, recovery, scratch and pool rules are one safety argument that should not be reviewed in halves. Three slices is the right count. The order lets the live runs prove the composition once. Each slice's red/green discipline is the policy's (narrow red first, widen after green, a failed import is not red), the existing TTL assertions are replaced rather than left endorsing superseded behavior, and the plan keeps the retry defaults, shard default and browser version out of scope. + +Execution placement is right throughout: edit, commit and push on the shared host; every run on Blaxel through the canonical runner from `tv-cleanup-coordinator-rose`; the driver as `npm test -- local --surface experiment:blaxel-harness` on that sandbox; private pool `rose-cleanup` for everything that touches workers; the pool deleted and re-provisioned around the fresh-provisioning acceptance because `ensure` is fresh-only. + +## Blocker: the final gate cannot complete as planned + +`plan.md:80-88` runs `npm run verify -- blaxel --pool rose-cleanup --shards 2 --allow-shard-count-override 1` on the two private workers and says this "replaces the proposal's earlier shared-`poc` final-run plan", with no reason given. It qualifies under the runner spec (`--shards` repartitions without narrowing, `test-runner.md:76`) and `verify` forwards `--pool`, `--shards` and the override (`scripts/test/cli.mjs:284`). It is still the wrong gate: + +- **Compute.** `test/timing-baseline.json` records Blaxel medians for 311 files totalling about 883 seconds, which is the 25-second mean per shard at 36 shards the Blaxel spec quotes. Two shards carry about 441 seconds each before checkout, dependency, pre-command builds, readiness and any e2e retry. `e2e:browser-app` alone is 376 seconds. +- **Ceiling.** The coordinator bounds each shard's process at the worker timeout: 480 seconds under the `cold` profile the canonical runner always passes (`run-blaxel-testshards.mjs`, `process.exec` timeout and `process.wait` maxWait). `verify` forwards no `--worker-timeout-seconds`, and neither the runner spec nor the plan proposes to add it. +- **No room to grow.** The tier's hard limit is 75 concurrent shards and `poc` holds 72 (`blaxel-testshards.md:25`), so a private pool cannot reach the standard count. + +The plan half-knows this: "two workers will be slower… and may expose a timeout/capacity limit; inspect the real plan/timings on Blaxel before the gate… report that blocker" (`:86`). The numbers are in the repository now; deferring the decision to the end of slice 3 puts the stall where it costs most. + +**To clear it:** run the final gate on `poc` at the standard 36 shards, from `tv-cleanup-coordinator-rose`, after the private pool's deletion acceptance has run and its sandboxes are gone. That is the converged proposal's arrangement, it is what the proposal's round 3 passed, it keeps Josh's rule (the ban is on the shared host, not the shared pool), and it is the ordinary production condition the new coordinator must work in: shared workers, other branches' runners, legacy locks and known leftovers. The private pool stays for the experiments that contaminate or kill things. If the author has a reason to avoid `poc` for the gate, the plan should state it so it can be weighed; none is visible to me. + +## Refinements (none blocking) + +1. **Whole-surface expansion on the private pool has the same ceiling.** `:19` sends surface expansion through `--pool rose-cleanup`; `--surface` takes the targeted path, one shard on one worker, so `e2e:browser-app` at 376 seconds plus setup is at the limit even on `poc`. For the large surfaces, expand with the planned selection path (`--package` or `--tag` with `--shards 2`) or leave whole-surface coverage to the gate; say which. +2. **`--lock-ttl-seconds` forwarding is a small runner change the plan should name as such.** `:19` says to ensure the canonical CLI forwards it for the driver. The runner spec enumerates no pass-through list, so this is code-governed and needs no spec line, but it is production runner code touched by slice 3 and belongs in that slice's implementation area with a sentence saying ordinary runs are unchanged. +3. **Slice 1 should say where the request inventory comes from.** "Retain a request inventory showing the built page" (`:35`) is the right observation; name the mechanism (Playwright's request events in the existing walk) so it is not read as a new harness. +4. **Record the tier arithmetic once.** Three private sandboxes beside `poc`'s 72 reach the 75-shard limit exactly; another agent's verify during the experiments would contend. The proposal already says to inspect actual usage before provisioning; the plan's execution section should repeat the one line so the implementer does not provision into a full tier. + +## Reviewer checklist + +- **Derivation.** Every slice points at proof anchors and spec sections, not at files alone; the implementation areas are named as areas, with the specs and proofs kept as the requirements. +- **Verification criteria and baselines.** Honest per slice; the final-gate criterion is the blocker above. +- **Placement.** No test, build, lint, type check, browser or server on the shared host; no shared `poc` workers for experiments; isolated-GitHub fixtures untouched. +- **Scope.** Proportionate: no sampler, archive, service or framework; production behavior untouched. +- **Status.** Accurate: no tests or implementation exist; unknowns listed (`:90-92`). + +## Evidence and limits + +- Read the plan in full, the proof and spec diffs since `3fa15d47`, the workflow's planning section, the policy's iteration and verification sections, the runner spec's qualification, verify and selector rules, and the runner's forwarding code; mapped each pending anchor to a slice. +- The timing figures are the baseline's recorded medians at 36 shards; they exclude setup phases and retries, so the two-shard estimate is a floor. +- Not established: the coordinator sandbox's local preflight capabilities for the driver surface; current tier occupancy. + +## Housekeeping + +- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel beyond a script that summed a committed JSON file into the scratchpad. `git fetch origin` updated remote-tracking refs. +- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-plan.md b/docs/working/test-harness-resource-cleanup-plan.md index 20fa1fba..9a590f23 100644 --- a/docs/working/test-harness-resource-cleanup-plan.md +++ b/docs/working/test-harness-resource-cleanup-plan.md @@ -1,6 +1,6 @@ # Test-harness resource cleanup: implementation plan -Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: ready for independent plan review; implementation has not begun. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. +Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: revised after [plan round 1](test-harness-resource-cleanup-plan-review-r1.md), ready for independent re-review; implementation has not begun. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. ## Slice judgment @@ -14,9 +14,9 @@ The specs and proofs remain the source of requirements. Paths below identify the Nothing in this plan runs tests, builds, lint, type checks, browsers or servers on the shared development host. Edit, commit and push each red or green revision there. Every execution uses that pushed revision on Blaxel through the canonical runner. A failed import, missing dependency, unavailable credential or setup failure is not the intended red result: establish that the narrow test reaches the missing behavior before counting it as red. -Use the named `tv-cleanup-coordinator-rose` sandbox outside all worker-pool labels and private pool `rose-cleanup`, with workers `tv-testshard-x64-rose-cleanup-01` and `-02`. Inventory, credentials, architecture, runtime and available capacity must be established before execution; the names are a plan, not evidence that hosts exist. The coordinator needs the repository's selected Node, dependencies and the local preflight capabilities required by the selected driver. The proposal owns ephemeral credential injection and credential-free logs. No test or experiment in this plan uses shared `poc` workers. +Use the named `tv-cleanup-coordinator-rose` sandbox outside all worker-pool labels and private pool `rose-cleanup`, with workers `tv-testshard-x64-rose-cleanup-01` and `-02`. Inventory, credentials, architecture, runtime and available capacity must be established before execution; the names are a plan, not evidence that hosts exist. The documented 72-worker shared pool plus these three sandboxes reaches the 75-slot tier limit exactly, leaving no allowance for other sandboxes; inspect actual workspace occupancy and limits before provisioning, and report insufficient capacity without changing shared workers. The coordinator needs the repository's selected Node, dependencies and the local preflight capabilities required by the selected driver. The proposal owns ephemeral credential injection and credential-free logs. Focused development runs and every worker experiment use the private pool. Ordinary full verification uses shared `poc` workers only after private qualification and deletion, as specified below. -Run ordinary focused tests from the coordinator as `npm test -- blaxel --pool rose-cleanup --file `, adding `--grep ` when narrowing a known failure. Surface expansion also names `--pool rose-cleanup`. Keep the default native and infrastructure retry budgets. Use a shortened existing recovery interval only for the declared abandoned-run experiments; it changes no production default. Ensure the canonical CLI forwards that existing coordinator option when implementing the live driver. +Run ordinary focused tests from the coordinator as `npm test -- blaxel --pool rose-cleanup --file `, adding `--grep ` when narrowing a known failure. Expand to affected complete files and directly affected helper callers there. A small whole-surface run also names `--pool rose-cleanup`; `--surface` always targets one worker even when `--shards` is supplied. Leave whole-surface coverage of large selections, including `e2e:browser-app`, to the standard-shard full gate below. Its recorded file medians alone total about 376 seconds, leaving little of the 480-second targeted-worker timeout for setup and retries. Keep the default native and infrastructure retry budgets. Use a shortened existing recovery interval only for the declared abandoned-run experiments; it changes no production default. Slice 3 adds the canonical forwarding needed for that explicit override. Run the live driver on the coordinator as `npm test -- local --surface experiment:blaxel-harness`. It calls the canonical Blaxel runner and pool manager for `rose-cleanup`; it is never itself a worker selection. Do not run the entire experiment suite or any isolated-GitHub-only lifecycle fixture. Before its fresh-provisioning acceptance, retrieve earlier private-run results and delete the now-idle private pool through the pool manager. The driver owns its coordinator subprocesses and pool cleanup, and reports any ownership or cleanup it cannot confirm. @@ -32,9 +32,9 @@ Convert only the persistence and executable-lifecycle cases in `packages/web/tes **Red/green and review criteria.** Implement the [build-lifetime contract and real worker seam](../../proofs/arch/testing-policy.md#Worker-owned builds). Begin with narrow red cases for early publication/failed preparation and outputs surviving worker teardown. Contract builders are the declared substitutes; the Playwright and Vitest seam must build real CLI/web output, use it, reuse it within a worker, retain it after case disposal and remove it after worker teardown. Exercise the absent-bundle copy failure within that same fixture, then a valid launch using the reusable build. Put cheap contracts in the ordinary repository suite and the real build/server crossing in the existing Node e2e surface, with fixture configs owned through the existing nested-runner arrangement. -For the five converted walks, make their built-loading boundary observable in the existing runs before changing page setup, then preserve all existing product assertions through the adaptation. Retain a request inventory showing the built page rather than the development-module graph; this adds no separate browser launch or source-text compliance scan. Confirm same-origin authenticated operation and restart/reload continuity. Grade the affected [theme](../../proofs/product/themes-and-appearance.md), [delivery](../../proofs/arch/themes/delivery.md), [artifact](../../proofs/product/artifacts.md) and [bridge](../../proofs/arch/artifact-frame/artifact-bridge.md) citations only after the real adapted walks pass. +For the five converted walks, make their built-loading boundary observable in the existing runs before changing page setup, then preserve all existing product assertions through the adaptation. Use Playwright request events in those walks to retain a request inventory showing the built page rather than the development-module graph; this adds no separate browser launch or source-text compliance scan. Confirm same-origin authenticated operation and restart/reload continuity. Grade the affected [theme](../../proofs/product/themes-and-appearance.md), [delivery](../../proofs/arch/themes/delivery.md), [artifact](../../proofs/product/artifacts.md) and [bridge](../../proofs/arch/artifact-frame/artifact-bridge.md) citations only after the real adapted walks pass. -Expand from the new helper tests and the five cases to both affected browser files, then the owning browser surfaces and directly affected helper callers where the change predicts impact. Preserve existing per-case process cleanup. This slice is green for its assigned obligations; killed-worker scratch recovery remains explicitly assigned to slice 3. +Expand from the new helper tests and the five cases to both affected browser files and directly affected helper callers where the change predicts impact. Whole large-surface regression coverage waits for the integrated gate under the execution rule above. Preserve existing per-case process cleanup. This slice is green for its assigned obligations; killed-worker scratch recovery remains explicitly assigned to slice 3. ## Slice 2 — bounded diagnostic capture and report transport @@ -56,6 +56,8 @@ The ordinary parent/child seams use the existing guarded local registry hook ins **Implementation area.** `scripts/run-blaxel-testshards.mjs`, `manage-blaxel-testshards.mjs`, their lease/discovery/dispatch helpers, the existing process owner/audit mechanisms, and the shared/targeted runner paths. Keep acquisition and entry checks usable before checkout; they cannot depend on code in a checkout another run may still own. Preserve the outer shard-attempt token across nested surface owners. A short OS-released guard serializes worker-local ownership changes; it is never held across report downloads. +In `scripts/test/cli.mjs`, forward an explicitly supplied `--lock-ttl-seconds` to the coordinator for the live driver's shortened recovery interval. This is a small runner implementation change, covered by the existing CLI delegation contracts and the live recovery run. Ordinary invocations without that option retain the spec's default interval; this does not add worker-timeout forwarding or change retry defaults. + Keep acquisition, renewal, dispatch registration, normal release, interruption and recovery in one lifecycle. Renew through setup, execution, teardown and retrieval; remove shell-exit unlock. Normal release follows confirmed descendants and verified downloads. The one expired-renewal procedure stops attributed owners, audits, records abandoned report loss and only then reuses the worker. Preserve the reviewed legacy behavior and fail-closed cases. Pool deletion refuses valid leases and removes dispatch eligibility before deletion; fresh provisioning publishes eligibility only after successful verification. No new service, caller-attestation system or mixed-version framework is needed. Give each shard attempt its specified private scratch and `TMPDIR`. Route uploads, plan, askpass and setup sidecars there, finalizing evidence outside it before removal. Pre-run cleanup removes only the two known direct `/tmp` prefixes and non-current attempt directories after the lease and clean owner audit. Preserve unrelated files, links' outside targets, current scratch and caches; reject unsafe roots/mounts. Stop processes before post-run removal and retain evidence/ownership when cleanup is uncertain. @@ -77,16 +79,18 @@ These discharge `^blaxel-t-retrieval`, `^blaxel-t-recovery-live`, `^blaxel-t-tmp After slice convergence, independently review the integrated specs, proofs, tests and code. Verify every new pending citation is either discharged with its real evidence or reported as unfinished; intentional fixture failures remain distinguishable from test-runner failures. Preserve the separate Settings development path, PR #9 reset, normal stale-owner protection and isolated-GitHub fixture restrictions. -Fetch the PR target, integrate it under the logical-merging workflow if needed, then commit and push the tree before the qualifying gate. Any target movement requires validation of the updated tree. Prepare fresh private workers after the live driver's pool-deletion case and run on `tv-cleanup-coordinator-rose`: +Fetch the PR target, integrate it under the logical-merging workflow if needed, then commit and push the tree before the qualifying gate. Any target movement requires validation of the updated tree. After private qualification, retain its reports and confirm the live driver's deletion of both private workers. Keep `tv-cleanup-coordinator-rose` to invoke ordinary full verification against the shared pool at the standard 36 shards: ```sh -npm run verify -- blaxel --pool rose-cleanup --shards 2 --allow-shard-count-override 1 +npm run verify -- blaxel --pool poc ``` -This runs every `all` shard index plus lint, type checking and package-manifest validation on Blaxel. The [publication-qualification rule](../../specs/arch/test-runner/test-runner.md#^test-publication-qualifying) permits a different partition count when the complete suite runs; it is not a narrowed gate. The two opt-in experiment surfaces remain separately qualified by the driver. Two workers will be slower than the standard pool and may expose a timeout/capacity limit; inspect the real plan/timings on Blaxel before the gate. If the named private capacity cannot complete it within the available runner limits, report that blocker to the supervisor rather than silently use the shared pool, omit shards or change retry budgets. +This runs every `all` shard index plus lint, type checking and package-manifest validation on Blaxel and has the standard [publication-qualifying](../../specs/arch/test-runner/test-runner.md#^test-publication-qualifying) shape. The two opt-in experiment surfaces remain separately qualified by the private driver. No deliberate interruption, contamination, pool deletion or provisioning targets `poc`; ordinary acquisition, cleanup and retrieval exercise the implemented lifecycle under its intended shared-pool conditions. Unavailable leases remain an infrastructure failure, not permission to steal a lock or alter the pool. + +The committed [timing baseline](../../test/timing-baseline.json), through 2026-09-29, records 882.506 seconds across 311 Blaxel file medians: about 441 seconds per worker if divided over two, before setup and retries. The canonical runner's cold profile permits 480 seconds per worker and does not forward a worker-timeout override. These historical totals are not a measured duration for the eventual tree, but they provide too little allowance to plan a two-worker full gate. Use the existing standard partition and pool instead of adding timeout controls or private capacity for this contribution. -This private full-gate arrangement replaces the proposal's earlier shared-`poc` final-run plan. Retrieve all evidence before deleting private hosts, and report cleanup that cannot be confirmed. Retain working documents while the PR is draft; perform pre-merge docs preparation only after implementation and integrated validation are complete. Josh's spec acceptance and the repository's required GitHub checks remain merge obligations; neither this plan nor an independent PASS claims they have happened. +Retrieve all evidence before deleting the dedicated coordinator, and report cleanup that cannot be confirmed. Retain working documents while the PR is draft; perform pre-merge docs preparation only after implementation and integrated validation are complete. Josh's spec acceptance and the repository's required GitHub checks remain merge obligations; neither this plan nor an independent PASS claims they have happened. ## Current limits -No new tests or implementation exist at this checkpoint and no execution has occurred. The coordinator's local preflight capabilities, credentials and capacity; actual provider behavior during renewal, label changes and interrupted dispatch; browser temporary placement; and private full-gate duration remain unmeasured. The plan carries those checks into the slices that can establish them. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. +No new tests or implementation exist at this checkpoint and no test or worker execution has occurred. The coordinator's local preflight capabilities, credentials and capacity; actual provider behavior during renewal, label changes and interrupted dispatch; browser temporary placement; and the implemented tree's full-gate duration remain unmeasured. The plan carries those checks into the slices that can establish them. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. diff --git a/docs/working/test-harness-resource-cleanup-proposal.md b/docs/working/test-harness-resource-cleanup-proposal.md index 426d8471..5370de28 100644 --- a/docs/working/test-harness-resource-cleanup-proposal.md +++ b/docs/working/test-harness-resource-cleanup-proposal.md @@ -233,11 +233,11 @@ Use a dedicated two-worker pool named **`rose-cleanup`**, with workers `tv-tests Re-run the converted acceptances without weakening their user-visible assertions. A recorded built-page request inventory should show that they no longer load the unbundled application graph. Validate restart continuity and authenticated navigation directly. Repeated use of one isolated worker should leave no generated runtime directories after cleanup and no growing sequence of resource reports. -For the final gate, provision a separate sandbox named **`tv-cleanup-coordinator-rose`**, excluded from testshard pool labels. It will fetch the committed, pushed branch and run `npm run verify -- blaxel`, so the invoking machine's lint/type-check/package-manifest phases also execute on Blaxel. The two experimental workers and coordinator are a plan, not currently provisioned capacity. Provision and verify their identities before starting validation; if unavailable, report blocked rather than run anything on this host or silently substitute GitHub execution. The user has already authorized outside-pool Blaxel hosts. +For the final gate, retain the separate sandbox named **`tv-cleanup-coordinator-rose`**, excluded from testshard pool labels. After private qualification and deletion of the two experimental workers, it will fetch the committed, pushed branch and run `npm run verify -- blaxel --pool poc` at the standard 36 shards, so the invoking machine's lint/type-check/package-manifest phases also execute on Blaxel. The two experimental workers and coordinator are a plan, not currently provisioned capacity. Provision and verify their identities before starting validation; if unavailable, report blocked rather than run anything on this host or silently substitute GitHub execution. The user has already authorized outside-pool Blaxel hosts. Supply the coordinator's credentials through the Blaxel SDK's `process.exec` environment: `BL_WORKSPACE` and an authorized `BL_API_KEY` for its outbound Blaxel calls, plus `BLAXEL_TV_GH_TOKEN` for repository access. The launching process reads the existing authorized secrets into memory and passes them directly in that environment object; do not interpolate their values into shell commands, uploaded scripts, output, or reports. The inspected SDK accepts the Blaxel pair without a CLI login/config file, and `readBlaxelGithubToken()` accepts `BLAXEL_TV_GH_TOKEN` when no `.blaxel-gh-token` file is present. Use that environment path in the fresh coordinator, with `GIT_ASKPASS` reading the environment for its initial fetch; the existing runner passes only the GitHub token onward to workers as `GH_TOKEN`. Do not copy local credential files or persist tokens in the coordinator's filesystem, sandbox configuration, or Git configuration. Confirm that the required credential is available and valid for the run before dispatch; its availability is not established by this proposal. End the credential-bearing process after downloading its reports and dispose the dedicated coordinator. This describes ephemeral process injection, not a claim about provider-internal retention. -The experiment plan adds at most three sandboxes. The spec's pool-size and tier examples are not a current capacity inventory: inspect actual workspace usage and provider limits before provisioning/dispatch. Do not claim slots are reserved or alter shared workers to obtain them. The [implementation plan](test-harness-resource-cleanup-plan.md#Integrated result and final validation) keeps all validation, including complete verification, on the named private hosts. After the live pool-deletion acceptance, provision fresh private workers and use the supported two-shard override for every `all` index. This preserves full selection while running more slowly than the standard pool. If the private capacity or available timeout blocks completion, report it rather than move validation to shared workers or omit coverage. Retrieve reports before deleting the private workers and coordinator. Record the exact tested revision, failures, recovered flakes, and any measurement limits. Ready-PR CI remains the repository's merge requirement; the contribution's execution evidence stays on Blaxel as instructed. +The experiment plan adds at most three sandboxes. The spec's pool-size and tier examples are not a current capacity inventory: inspect actual workspace usage and provider limits before provisioning/dispatch. Do not claim slots are reserved or alter shared workers to obtain them. The [implementation plan](test-harness-resource-cleanup-plan.md#Integrated result and final validation) keeps focused development and all worker experiments on the named private hosts, then uses the standard shared pool for ordinary full verification. The committed file medians leave inadequate setup/retry allowance for a two-worker full run within the canonical runner's 480-second worker timeout. Retrieve private reports before the pool-deletion acceptance and all remaining reports before deleting the coordinator. Record the exact tested revision, failures, recovered flakes, and any measurement limits. Ready-PR CI remains the repository's merge requirement; the contribution's execution evidence stays on Blaxel as instructed. Keep browser-version alignment, the default shard count, browser request limits, retry defaults, and production memory behavior outside this contribution. Forced GC is an investigation tool, not a proposed mitigation. Preserve main's demo-mode teardown fix; the drag flake has no established resource-exhaustion cause. @@ -249,4 +249,4 @@ Proposal round 2 at `d3570bf4`: FAIL because dead-coordinator lock recovery is u Proposal round 3 at `57a701d0`: **PASS**, relayed by the supervisor. The [review](test-harness-resource-cleanup-review-r3.md) is preserved unchanged. Its bounded refinements are applied here: fixed recovery interval, deletion of abandoned workers despite failed process cleanup, ordinary skipping of live legacy work, and trimmed receipt/provisioning language. The [response](test-harness-resource-cleanup-review-response.md#Round 3: convergence and spec derivation) records the judgments. The proposal and spec deltas have converged: spec round 1 at `cd8aa453` also passed. Its bounded refinements and the derived proof checkpoint are recorded in the response; independent proof review is next. -Proof round 1 at `9fd51f49`: **FAIL** on missing spec-owned placement for live qualification and deliberate-failure fixtures. The [review](test-harness-resource-cleanup-proof-review-r1.md) and [response](test-harness-resource-cleanup-review-response.md#Proof round 1: placement finding and refinements) retain that history. Proof round 2 at `3fa15d47`: **PASS**, including the placement delta. Its [report](test-harness-resource-cleanup-proof-review-r2.md) permits the bounded proof binding now applied. The [implementation plan](test-harness-resource-cleanup-plan.md) is ready for independent review; tests and implementation have not begun. No tests, builds, remote experiments, or worker operations were performed for this checkpoint. Unknowns are live provider behavior during failed dispatch/recovery, today's worker browser versions, actual `TMPDIR` placement and post-mitigation memory savings, current capacity/credential availability, private full-gate duration, and observed recovery timing. Recovery waits at most 20 minutes after the last renewal by default, then acquisition and bounded cleanup; failed cleanup prevents test reuse and explicit pool deletion remains available. Josh's spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. +Proof round 1 at `9fd51f49`: **FAIL** on missing spec-owned placement for live qualification and deliberate-failure fixtures. The [review](test-harness-resource-cleanup-proof-review-r1.md) and [response](test-harness-resource-cleanup-review-response.md#Proof round 1: placement finding and refinements) retain that history. Proof round 2 at `3fa15d47`: **PASS**, including the placement delta. Its [report](test-harness-resource-cleanup-proof-review-r2.md) permits the bounded proof binding now applied. Plan round 1 at `70f27ccb`: **FAIL** on the two-worker full-gate budget. The [review](test-harness-resource-cleanup-plan-review-r1.md) and [response](test-harness-resource-cleanup-review-response.md#Plan round 1: full-gate placement and refinements) record the correction to ordinary shared-pool verification and the bounded refinements. The [implementation plan](test-harness-resource-cleanup-plan.md) is ready for independent re-review; tests and implementation have not begun. No tests, builds, remote experiments, or worker operations were performed for this checkpoint. Unknowns are live provider behavior during failed dispatch/recovery, today's worker browser versions, actual `TMPDIR` placement and post-mitigation memory savings, current capacity/credential availability, the implemented tree's full-gate duration, and observed recovery timing. Recovery waits at most 20 minutes after the last renewal by default, then acquisition and bounded cleanup; failed cleanup prevents test reuse and explicit pool deletion remains available. Josh's spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 3ed611f8..9bd9a309 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -1,6 +1,6 @@ # Resource-cleanup proposal: review response -Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, and proof round 2 at `3fa15d47` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. The [round 2 response](#Proof round 2: convergence and implementation planning) records the bounded bindings/refinements and the three-slice plan ready for independent review. Earlier sections retain the response history. Reviewer: Claude Fable (Nina), arranged by the supervisor. +Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, and proof round 2 at `3fa15d47` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. Plan round 1 at `70f27ccb` is **FAIL** on the final-gate budget. The [current response](#Plan round 1: full-gate placement and refinements) records the correction and refinements; the three-slice plan is ready for independent re-review. Earlier sections retain the response history, including the superseded private full-gate arrangement. Reviewer: Claude Fable (Nina), arranged by the supervisor. Independent round 1 reviewed `1652787e`. The [full review](test-harness-resource-cleanup-review-r1.md) remains unchanged: FAIL on two blockers. The response committed at `99b7ed93` narrowed the work pending Josh's decisions. The following sections through round 2 record that history, not the current review verdict. @@ -171,3 +171,22 @@ Every test/build runs on a pushed revision on Blaxel. All worker-affecting valid Validation for this planning checkpoint is document/source inspection, exact review-copy comparison, Markdown index regeneration, and Git diff/whitespace inspection. No tests, builds, browser/server launches, provider experiments or worker operations ran. Implementation waits for the supervisor-arranged independent plan review. Josh's spec review remains at PR time. Primitive surfaced the already-recorded opt-in experiment choice after the prior turn. That was consistent context but added no new requirement. Its edit hook reported a truncated governing-set scan, so it did not establish a complete constraint check; no decision was overridden. This turn follows the settled direction; the slice ordering and private validation arrangement are contribution-scoped execution choices, so no new durable decision was recorded. + +## Plan round 1: full-gate placement and refinements + +The [complete review](test-harness-resource-cleanup-plan-review-r1.md) is preserved verbatim. Its one blocker is warranted: the two-worker full gate had no credible setup/retry allowance, and deferring that known problem until the end of implementation would only delay completion. The three slices, proof assignments and intermediate baselines remain unchanged. + +Read-only aggregation of `test/timing-baseline.json` confirms 311 Blaxel file medians totaling 882.506 seconds, or 441.253 seconds per worker when divided over two; `e2e:browser-app` contributes 376.485 seconds. These are historical measurements through 2026-09-29, not a measured duration or guaranteed timeout for the eventual tree. The coordinator's cold profile gives each worker process 480 seconds, including setup, and `scripts/test/cli.mjs` forwards no worker-timeout override. That is sufficient evidence to reject the two-worker gate budget without executing it. The documented tier has no allowance for a standard-size private pool beside the shared pool; actual occupancy still needs checking before provisioning. + +Restore ordinary full verification to `npm run verify -- blaxel --pool poc`, at the standard 36 shards, invoked from `tv-cleanup-coordinator-rose` after private qualification and deletion of both private workers. The previous all-private gate was my overly broad reading of the worker-experiment restriction, not a requirement from Josh. All tests, lint, type checks and builds still run on Blaxel at pushed revisions; every fault, interruption and cleanup experiment remains private. The shared pool sees only ordinary full verification through the implemented runner. This restores the converged proposal's validation arrangement without changing a spec, proof obligation, timeout, retry budget or shard default. + +| Refinement | Assessment and disposition | +| --- | --- | +| 1: large surface expansion | Accepted. Keep focused cases, complete affected files and directly affected helper callers on the private pool. Leave whole large surfaces, including `e2e:browser-app`, to the standard-shard integrated gate. State that `--surface` takes the one-worker targeted path even with `--shards`; do not imply the private pair splits it. | +| 2: name option forwarding | Accepted. Add `scripts/test/cli.mjs` and explicit `--lock-ttl-seconds` forwarding to slice 3's implementation area, with CLI delegation coverage and the existing live recovery run. Omitting the override retains the specified default. No new worker-timeout control. | +| 3: request inventory mechanism | Accepted. Use Playwright request events within the existing five walks. No extra launch or inventory harness. | +| 4: capacity arithmetic | Accepted as a planning bound, not a current inventory. The documented 72 shared workers plus two private workers and one coordinator reach 75 exactly. Check current workspace occupancy/limits before provisioning; report insufficient room without altering shared workers. | + +Validation for this revision is whole-review reading and exact-copy comparison, source/spec inspection, arithmetic over the committed timing JSON, and Git diff/whitespace checks. No test command, build, browser/server launch, provider call or worker operation ran. Implementation remains unstarted and waits for the supervisor-arranged plan re-review. Coordinator capabilities and current capacity remain unmeasured; Josh's spec acceptance remains due at PR time. + +Primitive's edit hook reported a truncated governing-set scan, so it did not establish a complete constraint check. No decision was overridden or reconciled. This revision follows the converged behavior and corrects contribution-scoped execution choices; it establishes no new durable team decision. From 564a8edb31767b7b6d1acb295d9c32846a4c1426 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:51:21 +0000 Subject: [PATCH 10/64] Add real worker lifetime seam before harness cleanup --- ...harness-resource-cleanup-plan-review-r2.md | 55 +++++++++++++++++ .../test-harness-resource-cleanup-plan.md | 6 +- ...arness-resource-cleanup-review-response.md | 10 ++- test/fixtures/worker-builds/exercise.ts | 50 +++++++++++++++ .../worker-builds/playwright.config.ts | 13 ++++ .../worker-builds/playwright.fixture.ts | 5 ++ test/fixtures/worker-builds/vitest.config.ts | 12 ++++ test/fixtures/worker-builds/vitest.fixture.ts | 5 ++ test/node/worker-build-lifetime.e2e.test.ts | 61 +++++++++++++++++++ 9 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 docs/working/test-harness-resource-cleanup-plan-review-r2.md create mode 100644 test/fixtures/worker-builds/exercise.ts create mode 100644 test/fixtures/worker-builds/playwright.config.ts create mode 100644 test/fixtures/worker-builds/playwright.fixture.ts create mode 100644 test/fixtures/worker-builds/vitest.config.ts create mode 100644 test/fixtures/worker-builds/vitest.fixture.ts create mode 100644 test/node/worker-build-lifetime.e2e.test.ts diff --git a/docs/working/test-harness-resource-cleanup-plan-review-r2.md b/docs/working/test-harness-resource-cleanup-plan-review-r2.md new file mode 100644 index 00000000..da4e808c --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-plan-review-r2.md @@ -0,0 +1,55 @@ +# Plan review, round 2: test-harness resource cleanup (PR #12, 04a6b275) + +**Verdict: PASS.** The final gate is back where it can complete: `npm run verify -- blaxel --pool poc` at the standard 36 shards, invoked from the dedicated coordinator sandbox after the private workers are qualified and deleted, with the private pool kept for every experiment that interrupts, contaminates or deletes anything. The four round 1 refinements are applied. Read as a whole, the plan still has three coherent slices, every pending proof obligation assigned, honest intermediate baselines, scope inside the converged specs, and no execution on the shared development host. Two small refinements follow, neither blocking. + +- Reviewed: the whole plan at `04a6b275`, the diff since `70f27ccb` for the plan, proposal and response, the copied round 1 review (identical to mine), and the runner code the plan's new statements rest on. No spec or proof changed in this revision. +- Branch state: contains `origin/main` (`a8e73fbf`). +- I ran nothing. Every check is a read of files in the checkout or of git history. + +## The blocker is cleared + +| Round 1 asked for | The plan now says | Checked | +| --- | --- | --- | +| A gate that can complete | `npm run verify -- blaxel --pool poc`, standard 36 shards, from `tv-cleanup-coordinator-rose`, after private qualification and deletion of both private workers (`plan.md:82-88`) | Publication-qualifying shape; `verify` forwards `--pool`; the coordinator sandbox is outside pool labels, so Josh's host rule holds. After the two private workers are gone, the tier holds `poc`'s 72 plus the coordinator. | +| A stated reason for leaving `poc`, or a return to it | The arithmetic is recorded (`:90`): 882.5 seconds of file medians, about 441 seconds per worker over two, against the 480-second cold worker timeout the canonical runner does not raise; no timeout control or private capacity is added for this contribution | Matches my own sum of `test/timing-baseline.json`; the response owns the earlier choice as an over-broad reading of the experiment restriction rather than a requirement. | +| `poc` used only for ordinary verification | "No deliberate interruption, contamination, pool deletion or provisioning targets `poc`… Unavailable leases remain an infrastructure failure, not permission to steal a lock or alter the pool" (`:88`) | This is the production condition the new coordinator must work in: shared workers, other branches' runners, legacy locks, known leftovers. The spec's transitional rule keeps other runs safe from it, and it from them. | + +## The refinements are applied + +1. **Large-surface expansion** (`:19`, `:37`): focused files and directly affected helper callers on the private pool; whole large surfaces, `e2e:browser-app` named, wait for the standard-shard gate. The new statement that `--surface` always takes the one-worker targeted path even with `--shards` is true: the targeted branch forces `--shards 1` (`scripts/test/cli.mjs:497`). +2. **`--lock-ttl-seconds` forwarding** is now in slice 3's implementation area as a runner change with ordinary invocations unchanged and no worker-timeout forwarding (`:59`). See refinement 1 below on its stated coverage. +3. **Request inventory** names Playwright request events inside the five existing walks (`:35`). +4. **Tier arithmetic** is in the execution section with the instruction to inspect occupancy before provisioning and to report rather than touch shared workers (`:17`). + +## The whole plan, re-read + +- **Slices.** Unchanged and still right: helpers and the five built-application walks; bounded diagnostics; the lease, dispatch, recovery, scratch, pool and live-qualification lifecycle kept together because it is one safety argument. +- **Obligations.** All 31 pending anchors assigned as in round 1; nothing moved. +- **Baselines.** Slice 1 green for its obligations with the killed-worker backstop assigned to slice 3; slice 2 explicit that the live diagnostics evidence and the lease/download race stay open; slice 3 resolves everything before completion (`:76`). +- **Sequencing on the private pool.** Slices 1 and 2 run focused tests on `rose-cleanup` under the existing coordinator; before slice 3's fresh-provisioning acceptance the idle pool is deleted through the manager so the driver's `^blaxel-t-pool-live` can create it fresh (`:21`); after the driver's deletion case nothing is re-provisioned and the gate moves to `poc`. The order is consistent with `ensure` being fresh-only. +- **Placement.** Edit, commit and push on the shared host; everything else on Blaxel from the coordinator sandbox; the driver as `npm test -- local --surface experiment:blaxel-harness` there; no isolated-GitHub fixtures; no experiment suite as a whole. +- **Evidence.** Red and green commit IDs, commands, run-directory paths and recovered retries retained per slice; the final gate reports the revision actually tested; docs prep deferred to after integrated validation; Josh's spec acceptance and required checks named as merge obligations, not claimed. +- **Limits.** Accurate (`:96`): the implemented tree's gate duration, coordinator capabilities and current tier occupancy are unmeasured and carried into the slices that can measure them. + +## Refinements (none blocking) + +1. **The forwarding's stated coverage overstates what exists.** `:59` says the `--lock-ttl-seconds` forwarding is "covered by the existing CLI delegation contracts". The existing coordinator tests in `test/repo/test-runner-provider.test.ts` and its neighbours read `run-blaxel-testshards.mjs` as source; none asserts the argument list `scripts/test/cli.mjs` hands the coordinator. The live recovery run shows the override arriving but cannot show that an ordinary invocation omits it. Slice 3 should add one small contract: an explicit `--lock-ttl-seconds` appears in the delegated arguments and is absent by default. It is a few lines beside the existing source assertions. +2. **Say what "the tier" counts.** `:17` treats the 75-shard hard limit as a sandbox count (72 + 3). The Blaxel spec phrases it as concurrent shards, and whether idle sandboxes count against it is not established. The sentence already tells the implementer to inspect actual occupancy and limits; adding "whether idle sandboxes count is unknown" keeps the first provisioning attempt from assuming either answer. + +## Reviewer checklist + +- **Derivation.** Slices point at proof anchors and spec sections; implementation areas are named as areas. +- **Verification criteria and baselines.** Honest per slice, and the final criterion is now achievable with the standard runner and pool. +- **Placement.** Nothing on the shared host; `poc` sees only ordinary verification; experiments private. +- **Scope.** Proportionate: no timeout controls, private capacity, sampler, archive, service or framework added; production behavior, retry defaults, shard default and browser version untouched. +- **Status.** Accurate: no tests or implementation exist; nothing has been executed. + +## Evidence and limits + +- Read the full plan, the three working-document diffs, and the runner code for the targeted path and verify pass-through; re-mapped the pending anchors; compared the copied review. +- Not established: current tier occupancy; the coordinator sandbox's local preflight capabilities for the driver surface. + +## Housekeeping + +- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel. `git fetch origin` updated remote-tracking refs. +- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-plan.md b/docs/working/test-harness-resource-cleanup-plan.md index 9a590f23..59c9b580 100644 --- a/docs/working/test-harness-resource-cleanup-plan.md +++ b/docs/working/test-harness-resource-cleanup-plan.md @@ -1,6 +1,6 @@ # Test-harness resource cleanup: implementation plan -Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: revised after [plan round 1](test-harness-resource-cleanup-plan-review-r1.md), ready for independent re-review; implementation has not begun. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. +Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; slice 1 implementation is underway. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. ## Slice judgment @@ -14,7 +14,7 @@ The specs and proofs remain the source of requirements. Paths below identify the Nothing in this plan runs tests, builds, lint, type checks, browsers or servers on the shared development host. Edit, commit and push each red or green revision there. Every execution uses that pushed revision on Blaxel through the canonical runner. A failed import, missing dependency, unavailable credential or setup failure is not the intended red result: establish that the narrow test reaches the missing behavior before counting it as red. -Use the named `tv-cleanup-coordinator-rose` sandbox outside all worker-pool labels and private pool `rose-cleanup`, with workers `tv-testshard-x64-rose-cleanup-01` and `-02`. Inventory, credentials, architecture, runtime and available capacity must be established before execution; the names are a plan, not evidence that hosts exist. The documented 72-worker shared pool plus these three sandboxes reaches the 75-slot tier limit exactly, leaving no allowance for other sandboxes; inspect actual workspace occupancy and limits before provisioning, and report insufficient capacity without changing shared workers. The coordinator needs the repository's selected Node, dependencies and the local preflight capabilities required by the selected driver. The proposal owns ephemeral credential injection and credential-free logs. Focused development runs and every worker experiment use the private pool. Ordinary full verification uses shared `poc` workers only after private qualification and deletion, as specified below. +Use the named `tv-cleanup-coordinator-rose` sandbox outside all worker-pool labels and private pool `rose-cleanup`, with workers `tv-testshard-x64-rose-cleanup-01` and `-02`. Inventory, credentials, architecture, runtime and available capacity must be established before execution. The documented 72-worker shared pool plus these three sandboxes would reach the documented 75-concurrent-shard limit if every sandbox counted; whether idle sandboxes count is unknown. Inspect actual workspace occupancy and limits before provisioning, and report insufficient capacity without changing shared workers. The coordinator needs the repository's selected Node, dependencies and the local preflight capabilities required by the selected driver. The proposal owns ephemeral credential injection and credential-free logs. Focused development runs and every worker experiment use the private pool. Ordinary full verification uses shared `poc` workers only after private qualification and deletion, as specified below. Run ordinary focused tests from the coordinator as `npm test -- blaxel --pool rose-cleanup --file `, adding `--grep ` when narrowing a known failure. Expand to affected complete files and directly affected helper callers there. A small whole-surface run also names `--pool rose-cleanup`; `--surface` always targets one worker even when `--shards` is supplied. Leave whole-surface coverage of large selections, including `e2e:browser-app`, to the standard-shard full gate below. Its recorded file medians alone total about 376 seconds, leaving little of the 480-second targeted-worker timeout for setup and retries. Keep the default native and infrastructure retry budgets. Use a shortened existing recovery interval only for the declared abandoned-run experiments; it changes no production default. Slice 3 adds the canonical forwarding needed for that explicit override. @@ -56,7 +56,7 @@ The ordinary parent/child seams use the existing guarded local registry hook ins **Implementation area.** `scripts/run-blaxel-testshards.mjs`, `manage-blaxel-testshards.mjs`, their lease/discovery/dispatch helpers, the existing process owner/audit mechanisms, and the shared/targeted runner paths. Keep acquisition and entry checks usable before checkout; they cannot depend on code in a checkout another run may still own. Preserve the outer shard-attempt token across nested surface owners. A short OS-released guard serializes worker-local ownership changes; it is never held across report downloads. -In `scripts/test/cli.mjs`, forward an explicitly supplied `--lock-ttl-seconds` to the coordinator for the live driver's shortened recovery interval. This is a small runner implementation change, covered by the existing CLI delegation contracts and the live recovery run. Ordinary invocations without that option retain the spec's default interval; this does not add worker-timeout forwarding or change retry defaults. +In `scripts/test/cli.mjs`, forward an explicitly supplied `--lock-ttl-seconds` to the coordinator for the live driver's shortened recovery interval. Add a small contract beside the existing CLI/provider assertions: the explicit option reaches the delegated arguments and is absent by default. The live recovery run proves it arrives at the worker. Ordinary invocations without that option retain the spec's default interval; this does not add worker-timeout forwarding or change retry defaults. Keep acquisition, renewal, dispatch registration, normal release, interruption and recovery in one lifecycle. Renew through setup, execution, teardown and retrieval; remove shell-exit unlock. Normal release follows confirmed descendants and verified downloads. The one expired-renewal procedure stops attributed owners, audits, records abandoned report loss and only then reuses the worker. Preserve the reviewed legacy behavior and fail-closed cases. Pool deletion refuses valid leases and removes dispatch eligibility before deletion; fresh provisioning publishes eligibility only after successful verification. No new service, caller-attestation system or mixed-version framework is needed. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 9bd9a309..414baa49 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -1,6 +1,6 @@ # Resource-cleanup proposal: review response -Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, and proof round 2 at `3fa15d47` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. Plan round 1 at `70f27ccb` is **FAIL** on the final-gate budget. The [current response](#Plan round 1: full-gate placement and refinements) records the correction and refinements; the three-slice plan is ready for independent re-review. Earlier sections retain the response history, including the superseded private full-gate arrangement. Reviewer: Claude Fable (Nina), arranged by the supervisor. +Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, proof round 2 at `3fa15d47`, and plan round 2 at `04a6b275` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. Slice 1 implementation is underway. Earlier sections retain the response history, including the superseded private full-gate arrangement. Reviewer: Claude Fable (Nina), arranged by the supervisor. Independent round 1 reviewed `1652787e`. The [full review](test-harness-resource-cleanup-review-r1.md) remains unchanged: FAIL on two blockers. The response committed at `99b7ed93` narrowed the work pending Josh's decisions. The following sections through round 2 record that history, not the current review verdict. @@ -190,3 +190,11 @@ Restore ordinary full verification to `npm run verify -- blaxel --pool poc`, at Validation for this revision is whole-review reading and exact-copy comparison, source/spec inspection, arithmetic over the committed timing JSON, and Git diff/whitespace checks. No test command, build, browser/server launch, provider call or worker operation ran. Implementation remains unstarted and waits for the supervisor-arranged plan re-review. Coordinator capabilities and current capacity remain unmeasured; Josh's spec acceptance remains due at PR time. Primitive's edit hook reported a truncated governing-set scan, so it did not establish a complete constraint check. No decision was overridden or reconciled. This revision follows the converged behavior and corrects contribution-scoped execution choices; it establishes no new durable team decision. + +## Plan round 2: convergence and slice 1 + +The [complete PASS report](test-harness-resource-cleanup-plan-review-r2.md) is preserved unchanged. Accept its two bounded refinements: slice 3 adds an explicit supplied/default CLI-argument contract for `--lock-ttl-seconds`, and capacity planning no longer equates the documented concurrent-shard limit with a count of idle sandboxes. No slice boundary or proof obligation changes. + +Read-only provider inventory found 143 deployed `poc` sandboxes and none of the named private hosts. The documented 72-worker inventory is therefore not current. Blaxel accepted creation of `tv-cleanup-coordinator-rose`; the canonical pool manager, running there at pushed `04a6b275`, provisioned `tv-testshard-x64-rose-cleanup-01` and `-02`. Each uses the named Playwright image, 8192 MB and Node 24.22.1/npm 11.19.1, with x64/Linux verified on the workers. API-key authentication and the GitHub credential are available; values were passed only through the SDK process environment. The provider accepted this capacity, but its account-specific quota and idle-sandbox accounting remain unknown. No shared worker was modified. + +The first red checkpoint adds a real Playwright/Vitest worker fixture and parent seam. Each worker uses real source CLI and versioned web output in two cases, observes reuse after per-case cleanup, and leaves its outputs for the parent to inspect after normal teardown. An absent bundle exercises failed-launch private-copy cleanup; a later worker must receive different paths. Nested fixture retries are zero to observe exactly two cases; the canonical parent retains its default retry budget. Test execution has not yet occurred at this checkpoint. All subsequent commands and evidence are recorded below. diff --git a/test/fixtures/worker-builds/exercise.ts b/test/fixtures/worker-builds/exercise.ts new file mode 100644 index 00000000..39fd6fef --- /dev/null +++ b/test/fixtures/worker-builds/exercise.ts @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import { appendFileSync, existsSync, readFileSync, readdirSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { disposeAllProductServers, launchProductServer } from "../../helpers/product-server.ts"; +import { buildVersionedWebBundle } from "../../helpers/versioned-web-bundle.ts"; + +export async function exerciseWorkerBuilds(caseNumber: number): Promise { + const temporaryRoot = os.tmpdir(); + const recordPath = process.env.TV_WORKER_BUILD_RECORD; + assert.ok(recordPath, "the parent supplies a private observation file"); + + if (caseNumber === 1) { + await assert.rejects(launchProductServer({ + onboardingBundle: path.join(temporaryRoot, "absent-bundle"), + env: { VITEST: "" }, + }), /ENOENT/); + } + + const server = await launchProductServer({ env: { VITEST: "" } }); + let webBundle: string; + try { + const response = await fetch(server.serverURL); + assert.equal(response.status, 200); + const document = await response.text(); + const script = document.match(/src="([^"]+\.js)"/); + assert.ok(script, "the source CLI serves the built application"); + const asset = await fetch(new URL(script[1]!, server.serverURL)); + assert.equal(asset.status, 200); + assert.ok((await asset.text()).length > 0); + + webBundle = await buildVersionedWebBundle("worker-lifetime-fixture"); + const builtDocument = readFileSync(path.join(webBundle, "index.html"), "utf8"); + const builtScript = builtDocument.match(/src="([^"]+\.js)"/); + assert.ok(builtScript, "the real versioned builder emits an application entry"); + assert.ok(readFileSync(path.join(webBundle, builtScript[1]!)).length > 0); + } finally { + await server.dispose(); + await disposeAllProductServers(); + } + + const sourceDirectories = readdirSync(temporaryRoot) + .filter((entry) => entry.startsWith("television-source-cli-e2e-")) + .map((entry) => path.join(temporaryRoot, entry)); + assert.equal(sourceDirectories.length, 1, "one source build is shared across cases"); + assert.ok(existsSync(path.join(sourceDirectories[0]!, "tv.cjs"))); + assert.ok(existsSync(path.join(webBundle!, "index.html")), "case cleanup preserves the shared bundle"); + assert.equal(existsSync(server.home), false, "case cleanup removes its private home"); + appendFileSync(recordPath, `${JSON.stringify({ caseNumber, pid: process.pid, source: sourceDirectories[0], webBundle })}\n`); +} diff --git a/test/fixtures/worker-builds/playwright.config.ts b/test/fixtures/worker-builds/playwright.config.ts new file mode 100644 index 00000000..76e18e19 --- /dev/null +++ b/test/fixtures/worker-builds/playwright.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from "@playwright/test"; + +export default defineConfig({ + testDir: import.meta.dirname, + testMatch: "playwright.fixture.ts", + workers: 1, + fullyParallel: false, + // The parent acceptance owns retries; this fixture observes exactly two cases. + retries: 0, + timeout: 90_000, + reporter: "line", + outputDir: process.env.TV_WORKER_BUILD_OUTPUT, +}); diff --git a/test/fixtures/worker-builds/playwright.fixture.ts b/test/fixtures/worker-builds/playwright.fixture.ts new file mode 100644 index 00000000..44be09a9 --- /dev/null +++ b/test/fixtures/worker-builds/playwright.fixture.ts @@ -0,0 +1,5 @@ +import { test } from "../../helpers/playwright.ts"; +import { exerciseWorkerBuilds } from "./exercise.ts"; + +test("first case prepares usable builds", async () => exerciseWorkerBuilds(1)); +test("second case reuses the worker builds", async () => exerciseWorkerBuilds(2)); diff --git a/test/fixtures/worker-builds/vitest.config.ts b/test/fixtures/worker-builds/vitest.config.ts new file mode 100644 index 00000000..a48fd304 --- /dev/null +++ b/test/fixtures/worker-builds/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["test/fixtures/worker-builds/vitest.fixture.ts"], + maxWorkers: 1, + fileParallelism: false, + // The parent acceptance owns retries; this fixture observes exactly two cases. + retry: 0, + testTimeout: 90_000, + }, +}); diff --git a/test/fixtures/worker-builds/vitest.fixture.ts b/test/fixtures/worker-builds/vitest.fixture.ts new file mode 100644 index 00000000..4934dd18 --- /dev/null +++ b/test/fixtures/worker-builds/vitest.fixture.ts @@ -0,0 +1,5 @@ +import { test } from "vitest"; +import { exerciseWorkerBuilds } from "./exercise.ts"; + +test("first case prepares usable builds", async () => exerciseWorkerBuilds(1)); +test("second case reuses the worker builds", async () => exerciseWorkerBuilds(2)); diff --git a/test/node/worker-build-lifetime.e2e.test.ts b/test/node/worker-build-lifetime.e2e.test.ts new file mode 100644 index 00000000..405787ba --- /dev/null +++ b/test/node/worker-build-lifetime.e2e.test.ts @@ -0,0 +1,61 @@ +import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import { expect, test } from "vitest"; +import { spawnOwnedProcess } from "../helpers/owned-process.ts"; + +const require = createRequire(import.meta.url); +const root = path.resolve(import.meta.dirname, "../.."); +type Observation = { caseNumber: number; pid: number; source: string; webBundle: string }; + +// proofs/arch/testing-policy.md#^testing-t-build-worker-seam +test.each(["playwright", "vitest"] as const)("%s workers reuse builds and remove them at teardown", async (runner) => { + let previous: Observation | undefined; + for (let worker = 0; worker < 2; worker += 1) { + const temporaryRoot = mkdtempSync(path.join(os.tmpdir(), "television-worker-build-seam-")); + const record = path.join(temporaryRoot, "observations.jsonl"); + const cli = runner === "playwright" + ? require.resolve("@playwright/test/cli") + : path.join(path.dirname(require.resolve("vitest/package.json")), "vitest.mjs"); + const owned = spawnOwnedProcess(process.execPath, [ + cli, runner === "playwright" ? "test" : "run", + "--config", `test/fixtures/worker-builds/${runner}.config.ts`, + ], { + cwd: root, + env: { ...process.env, TMPDIR: temporaryRoot, TV_WORKER_BUILD_RECORD: record, TV_WORKER_BUILD_OUTPUT: path.join(temporaryRoot, "results") }, + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + owned.child.stdout!.on("data", (chunk: Buffer) => { output += chunk.toString(); }); + owned.child.stderr!.on("data", (chunk: Buffer) => { output += chunk.toString(); }); + try { + const exitCode = await new Promise((resolve, reject) => { + owned.child.once("error", reject); + owned.child.once("close", resolve); + }); + expect(exitCode, output).toBe(0); + const observations = readFileSync(record, "utf8").trim().split("\n").map((line) => JSON.parse(line) as Observation); + expect(observations).toHaveLength(2); + const [first, second] = observations as [Observation, Observation]; + expect(first.caseNumber).toBe(1); + expect(second).toEqual({ ...first, caseNumber: 2 }); + if (previous) { + expect(first.source).not.toBe(previous.source); + expect(first.webBundle).not.toBe(previous.webBundle); + } + previous = first; + + const remaining = readdirSync(temporaryRoot).filter((name) => + name.startsWith("television-source-cli-e2e-") || + name.startsWith("television-web-bundle-") || + name.startsWith("television-product-e2e-") + ); + expect(remaining, `${runner} left worker builds or failed-launch copies after teardown`).toEqual([]); + } finally { + const cleanup = await owned.dispose(); + expect(cleanup.outcome, output).not.toBe("survived"); + rmSync(temporaryRoot, { recursive: true, force: true }); + } + } +}); From 74485d227e43f0000cce6775c71af80568e094b2 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:58:58 +0000 Subject: [PATCH 11/64] Generate nested worker fixture configs within their acceptance --- .../fixtures/worker-builds/playwright.config.ts | 13 ------------- test/fixtures/worker-builds/vitest.config.ts | 12 ------------ test/node/worker-build-lifetime.e2e.test.ts | 17 ++++++++++++++--- 3 files changed, 14 insertions(+), 28 deletions(-) delete mode 100644 test/fixtures/worker-builds/playwright.config.ts delete mode 100644 test/fixtures/worker-builds/vitest.config.ts diff --git a/test/fixtures/worker-builds/playwright.config.ts b/test/fixtures/worker-builds/playwright.config.ts deleted file mode 100644 index 76e18e19..00000000 --- a/test/fixtures/worker-builds/playwright.config.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { defineConfig } from "@playwright/test"; - -export default defineConfig({ - testDir: import.meta.dirname, - testMatch: "playwright.fixture.ts", - workers: 1, - fullyParallel: false, - // The parent acceptance owns retries; this fixture observes exactly two cases. - retries: 0, - timeout: 90_000, - reporter: "line", - outputDir: process.env.TV_WORKER_BUILD_OUTPUT, -}); diff --git a/test/fixtures/worker-builds/vitest.config.ts b/test/fixtures/worker-builds/vitest.config.ts deleted file mode 100644 index a48fd304..00000000 --- a/test/fixtures/worker-builds/vitest.config.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { defineConfig } from "vitest/config"; - -export default defineConfig({ - test: { - include: ["test/fixtures/worker-builds/vitest.fixture.ts"], - maxWorkers: 1, - fileParallelism: false, - // The parent acceptance owns retries; this fixture observes exactly two cases. - retry: 0, - testTimeout: 90_000, - }, -}); diff --git a/test/node/worker-build-lifetime.e2e.test.ts b/test/node/worker-build-lifetime.e2e.test.ts index 405787ba..21020fd7 100644 --- a/test/node/worker-build-lifetime.e2e.test.ts +++ b/test/node/worker-build-lifetime.e2e.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; import os from "node:os"; import path from "node:path"; @@ -15,15 +15,26 @@ test.each(["playwright", "vitest"] as const)("%s workers reuse builds and remove for (let worker = 0; worker < 2; worker += 1) { const temporaryRoot = mkdtempSync(path.join(os.tmpdir(), "television-worker-build-seam-")); const record = path.join(temporaryRoot, "observations.jsonl"); + // Nested fixtures belong to this acceptance, not independently runnable surfaces. + // Their native configs are temporary, as in the runner-reporting acceptance. + const config = path.join(temporaryRoot, `${runner}.config.mjs`); + writeFileSync(config, `export default ${JSON.stringify(runner === "playwright" ? { + testDir: path.join(root, "test/fixtures/worker-builds"), + testMatch: "playwright.fixture.ts", workers: 1, fullyParallel: false, + retries: 0, timeout: 90_000, reporter: "line", outputDir: path.join(temporaryRoot, "results"), + } : { test: { + include: ["test/fixtures/worker-builds/vitest.fixture.ts"], maxWorkers: 1, + fileParallelism: false, retry: 0, testTimeout: 90_000, + } })};\n`); const cli = runner === "playwright" ? require.resolve("@playwright/test/cli") : path.join(path.dirname(require.resolve("vitest/package.json")), "vitest.mjs"); const owned = spawnOwnedProcess(process.execPath, [ cli, runner === "playwright" ? "test" : "run", - "--config", `test/fixtures/worker-builds/${runner}.config.ts`, + "--config", config, ], { cwd: root, - env: { ...process.env, TMPDIR: temporaryRoot, TV_WORKER_BUILD_RECORD: record, TV_WORKER_BUILD_OUTPUT: path.join(temporaryRoot, "results") }, + env: { ...process.env, TMPDIR: temporaryRoot, TV_WORKER_BUILD_RECORD: record }, stdio: ["ignore", "pipe", "pipe"], }); let output = ""; From 2460d06a1efcf13b2c9790636f5174bcded1531f Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:00:58 +0000 Subject: [PATCH 12/64] Specify failed and pending worker build cleanup contracts --- test/helpers/worker-builds.ts | 24 +++++++++++ test/repo/worker-builds.test.ts | 74 +++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 test/helpers/worker-builds.ts create mode 100644 test/repo/worker-builds.test.ts diff --git a/test/helpers/worker-builds.ts b/test/helpers/worker-builds.ts new file mode 100644 index 00000000..c83dcfd9 --- /dev/null +++ b/test/helpers/worker-builds.ts @@ -0,0 +1,24 @@ +import { mkdtempSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +/** Owns reusable preparation for one runner worker. */ +export class WorkerBuilds { + private readonly builds = new Map>(); + + constructor(private readonly prefix: string, private readonly parent = os.tmpdir()) {} + + get(key: string, prepare: (directory: string) => void | Promise): Promise { + let build = this.builds.get(key); + if (!build) { + const directory = mkdtempSync(path.join(this.parent, this.prefix)); + build = Promise.resolve().then(() => prepare(directory)).then(() => directory); + this.builds.set(key, build); + } + return build; + } + + async dispose(): Promise { + await Promise.allSettled(this.builds.values()); + } +} diff --git a/test/repo/worker-builds.test.ts b/test/repo/worker-builds.test.ts new file mode 100644 index 00000000..8a127b26 --- /dev/null +++ b/test/repo/worker-builds.test.ts @@ -0,0 +1,74 @@ +import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { WorkerBuilds } from "../helpers/worker-builds.ts"; + +const roots: string[] = []; +function fixture(): { root: string; builds: WorkerBuilds } { + const root = mkdtempSync(path.join(os.tmpdir(), "worker-build-contract-")); + roots.push(root); + return { root, builds: new WorkerBuilds("output-", root) }; +} +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +// proofs/arch/testing-policy.md#^testing-t-build-lifetime +test("concurrent and completed requests reuse a build while version keys remain distinct", async () => { + const { root, builds } = fixture(); + const ready = Promise.withResolvers(); + let calls = 0; + const first = builds.get("v1", async (directory) => { + calls += 1; + writeFileSync(path.join(directory, "entry.js"), "version one"); + await ready.promise; + }); + const concurrent = builds.get("v1", () => { throw new Error("prepared twice"); }); + expect(concurrent).toBe(first); + ready.resolve(); + const directory = await first; + expect(await builds.get("v1", () => { throw new Error("prepared again"); })).toBe(directory); + const second = await builds.get("v2", (output) => { writeFileSync(path.join(output, "entry.js"), "version two"); }); + expect(second).not.toBe(directory); + expect(calls).toBe(1); + expect(existsSync(path.join(directory, "entry.js"))).toBe(true); + await builds.dispose(); + expect(readdirSync(root)).toEqual([]); +}); + +test("failed preparation removes partial output and permits a fresh build", async () => { + const { root, builds } = fixture(); + let failedDirectory = ""; + await expect(builds.get("source", (directory) => { + failedDirectory = directory; + writeFileSync(path.join(directory, "partial.js"), "incomplete"); + throw new Error("asset preparation failed"); + })).rejects.toThrow("asset preparation failed"); + expect(readdirSync(root)).toEqual([]); + const directory = await builds.get("source", (output) => { writeFileSync(path.join(output, "entry.js"), "complete"); }); + expect(directory).not.toBe(failedDirectory); + expect(existsSync(path.join(directory, "entry.js"))).toBe(true); + await builds.dispose(); +}); + +test.each([false, true])("teardown waits for pending preparation, including rejection (%s)", async (reject) => { + const { root, builds } = fixture(); + const started = Promise.withResolvers(); + const ready = Promise.withResolvers(); + const build = builds.get("pending", async (directory) => { + writeFileSync(path.join(directory, "partial.js"), "pending"); + started.resolve(directory); + await ready.promise; + if (reject) throw new Error("late preparation failure"); + }); + const settled = Promise.allSettled([build]); + const directory = await started.promise; + let disposed = false; + const cleanup = builds.dispose().then(() => { disposed = true; }); + await Promise.resolve(); + expect(disposed).toBe(false); + expect(existsSync(directory)).toBe(true); + ready.resolve(); + expect((await settled)[0]!.status).toBe(reject ? "rejected" : "fulfilled"); + await cleanup; + expect(readdirSync(root)).toEqual([]); +}); From 8bfd952f577b3826822ec9dc1c27ad14085cf27d Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:04:01 +0000 Subject: [PATCH 13/64] Dispose reusable builds at runner worker teardown --- packages/web/test/foundation-copy.test.ts | 3 +- test/fixtures/worker-builds/exercise.ts | 2 + test/fixtures/worker-builds/vitest.fixture.ts | 2 +- test/helpers/playwright.ts | 14 ++- test/helpers/product-server.ts | 98 ++++++++++--------- test/helpers/versioned-web-bundle.ts | 22 ++--- test/helpers/vitest.ts | 16 +++ test/helpers/worker-builds.ts | 29 +++++- test/node/acp-home-context.test.ts | 3 +- test/node/product-server-proxy.e2e.test.ts | 9 +- test/repo/build-config-integrity.test.ts | 3 +- 11 files changed, 128 insertions(+), 73 deletions(-) create mode 100644 test/helpers/vitest.ts diff --git a/packages/web/test/foundation-copy.test.ts b/packages/web/test/foundation-copy.test.ts index 0fb0a976..ca180f14 100644 --- a/packages/web/test/foundation-copy.test.ts +++ b/packages/web/test/foundation-copy.test.ts @@ -1,6 +1,7 @@ import { globSync, readFileSync, statSync } from "node:fs"; import path from "node:path"; -import { describe, expect, test } from "vitest"; +import { describe, expect } from "vitest"; +import { test } from "../../../test/helpers/vitest.ts"; import { enumerateTrackedPaths } from "../../../scripts/test/file-inventory.mjs"; import { buildVersionedWebBundle } from "../../../test/helpers/versioned-web-bundle.ts"; import { diff --git a/test/fixtures/worker-builds/exercise.ts b/test/fixtures/worker-builds/exercise.ts index 39fd6fef..9804d025 100644 --- a/test/fixtures/worker-builds/exercise.ts +++ b/test/fixtures/worker-builds/exercise.ts @@ -15,6 +15,8 @@ export async function exerciseWorkerBuilds(caseNumber: number): Promise { onboardingBundle: path.join(temporaryRoot, "absent-bundle"), env: { VITEST: "" }, }), /ENOENT/); + assert.deepEqual(readdirSync(temporaryRoot).filter((entry) => entry.startsWith("television-product-e2e-")), [], + "the failed launch removes its home and private runtime before another launch"); } const server = await launchProductServer({ env: { VITEST: "" } }); diff --git a/test/fixtures/worker-builds/vitest.fixture.ts b/test/fixtures/worker-builds/vitest.fixture.ts index 4934dd18..8bc4c7a4 100644 --- a/test/fixtures/worker-builds/vitest.fixture.ts +++ b/test/fixtures/worker-builds/vitest.fixture.ts @@ -1,4 +1,4 @@ -import { test } from "vitest"; +import { test } from "../../helpers/vitest.ts"; import { exerciseWorkerBuilds } from "./exercise.ts"; test("first case prepares usable builds", async () => exerciseWorkerBuilds(1)); diff --git a/test/helpers/playwright.ts b/test/helpers/playwright.ts index d2e51024..39bef2cb 100644 --- a/test/helpers/playwright.ts +++ b/test/helpers/playwright.ts @@ -1,10 +1,20 @@ import { expect, test as base } from "@playwright/test"; -import { disposeAllProductServers } from "./product-server.ts"; +import { disposeAllProductServers, disposeSourceCLIBuilds } from "./product-server.ts"; +import { disposeVersionedWebBuilds } from "./versioned-web-bundle.ts"; export { expect }; /** Playwright test registration with per-test cleanup for product servers, including interrupted launches, and their app proxies. */ -export const test = base.extend<{ productServerCleanup: void }>({ +export const test = base.extend<{ productServerCleanup: void }, { workerBuildCleanup: void }>({ + workerBuildCleanup: [async ({}, use) => { + try { + await use(); + } finally { + // Keep runtime files if a live server could not be stopped. + await disposeAllProductServers(); + await Promise.all([disposeSourceCLIBuilds(), disposeVersionedWebBuilds()]); + } + }, { scope: "worker", auto: true }], productServerCleanup: [async ({}, use) => { try { await use(); diff --git a/test/helpers/product-server.ts b/test/helpers/product-server.ts index 15a3f3cb..db6902a3 100644 --- a/test/helpers/product-server.ts +++ b/test/helpers/product-server.ts @@ -9,11 +9,13 @@ import { fileURLToPath } from "node:url"; import { parseConnectURL } from "@telepath-computer/television-shared"; import { spawnOwnedProcess, type OwnedProcess } from "./owned-process.ts"; import { writeHomeConfig } from "./television-home.ts"; +import { WorkerBuilds } from "./worker-builds.ts"; const HERE = path.dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = path.resolve(HERE, "../.."); const CLI_BUILD_SCRIPT = path.join(REPO_ROOT, "packages/cli/build.mjs"); -let sourceCLIEntry: string | null = null; +const sourceCLIBuilds = new WorkerBuilds("television-source-cli-e2e-"); +export const disposeSourceCLIBuilds = (): Promise => sourceCLIBuilds.dispose(); const serverStartupTimeoutMs = 30_000; export interface ProductServer { @@ -64,16 +66,15 @@ export interface LaunchProductServerOptions { env?: Record; } -function getSourceCLIEntry(): string { - if (sourceCLIEntry) return sourceCLIEntry; - const buildDir = mkdtempSync(path.join(os.tmpdir(), "television-source-cli-e2e-")); - sourceCLIEntry = path.join(buildDir, "tv.cjs"); - execFileSync(process.execPath, [CLI_BUILD_SCRIPT, "--outfile", sourceCLIEntry], { - cwd: REPO_ROOT, - stdio: "pipe", +async function getSourceCLIEntry(): Promise { + const buildDir = await sourceCLIBuilds.get("source", (directory) => { + execFileSync(process.execPath, [CLI_BUILD_SCRIPT, "--outfile", path.join(directory, "tv.cjs")], { + cwd: REPO_ROOT, + stdio: "pipe", + }); + installProductRuntimeAssets(directory); }); - installProductRuntimeAssets(buildDir); - return sourceCLIEntry; + return path.join(buildDir, "tv.cjs"); } /** @@ -81,23 +82,27 @@ function getSourceCLIEntry(): string { * The shared cached layout stays bundle-free so ordinary launches keep the * missing-bundle behavior. */ -function getCLIEntryWithBundles(options: { +function getCLIEntryWithBundles(sharedEntry: string, options: { onboardingBundle?: string; bundledThemesBundle?: string; }): { cliEntry: string; dispose: () => void } { - const sharedEntry = getSourceCLIEntry(); const runDir = mkdtempSync(path.join(os.tmpdir(), "television-product-e2e-bundle-")); - cpSync(path.dirname(sharedEntry), runDir, { recursive: true }); - if (options.onboardingBundle !== undefined) { - cpSync(options.onboardingBundle, path.join(runDir, "onboarding"), { recursive: true }); - } - if (options.bundledThemesBundle !== undefined) { - cpSync(options.bundledThemesBundle, path.join(runDir, "themes"), { recursive: true }); + try { + cpSync(path.dirname(sharedEntry), runDir, { recursive: true }); + if (options.onboardingBundle !== undefined) { + cpSync(options.onboardingBundle, path.join(runDir, "onboarding"), { recursive: true }); + } + if (options.bundledThemesBundle !== undefined) { + cpSync(options.bundledThemesBundle, path.join(runDir, "themes"), { recursive: true }); + } + return { + cliEntry: path.join(runDir, path.basename(sharedEntry)), + dispose: () => rmSync(runDir, { recursive: true, force: true }), + }; + } catch (error) { + rmSync(runDir, { recursive: true, force: true }); + throw error; } - return { - cliEntry: path.join(runDir, path.basename(sharedEntry)), - dispose: () => rmSync(runDir, { recursive: true, force: true }), - }; } function installProductRuntimeAssets(buildDir: string): void { @@ -157,22 +162,12 @@ export async function disposeAllProductServers(): Promise { export async function launchProductServer(options: LaunchProductServerOptions = {}): Promise { const createdHome = options.home === undefined; const home = options.home ?? mkdtempSync(path.join(os.tmpdir(), "television-product-e2e-")); - // The test-runner recipe for spawned `tv serve`: config port 0, with the - // acquired port read from startup output and passed to client commands. - writeHomeConfig(home, options.auth ? { port: 0 } : { port: 0, auth: false }); const { TELEVISION_ACP_AGENT: _ignoredAgent, ...inheritedEnv } = process.env; const env = { ...inheritedEnv, ...options.env }; - let bundledRuntime = options.onboardingBundle !== undefined || options.bundledThemesBundle !== undefined - ? getCLIEntryWithBundles({ - ...(options.onboardingBundle !== undefined - ? { onboardingBundle: options.onboardingBundle } - : {}), - ...(options.bundledThemesBundle !== undefined - ? { bundledThemesBundle: options.bundledThemesBundle } - : {}), - }) - : undefined; - const cliEntry = (): string => bundledRuntime?.cliEntry ?? getSourceCLIEntry(); + let sharedEntry: string; + let preparation: Promise | undefined; + let bundledRuntime: ReturnType | undefined; + const cliEntry = (): string => bundledRuntime?.cliEntry ?? sharedEntry; const proxies = new Map(); let current: { process: ProductServerProcess; serverURL: string } | null = null; let pendingProcess: ProductServerProcess | null = null; @@ -201,23 +196,32 @@ export async function launchProductServer(options: LaunchProductServerOptions = const dispose: ProductServerDisposer = () => { disposalRequested = true; disposalPromise ??= (async () => { - try { - await Promise.all([...proxies.values()].map((proxy) => proxy.dispose())); - const processes = new Set([current?.process, pendingProcess].filter((process): process is ProductServerProcess => process != null)); - await Promise.all([...processes].map(stopProcess)); - bundledRuntime?.dispose(); - if (options.cleanupHome ?? createdHome) { - rmSync(home, { recursive: true, force: true }); - } - } finally { - activeProductServerDisposers.delete(dispose); + // A disposal requested during preparation must not race removal of its home. + await preparation?.catch(() => undefined); + await Promise.all([...proxies.values()].map((proxy) => proxy.dispose())); + const processes = new Set([current?.process, pendingProcess].filter((process): process is ProductServerProcess => process != null)); + await Promise.all([...processes].map(stopProcess)); + bundledRuntime?.dispose(); + if (options.cleanupHome ?? createdHome) { + rmSync(home, { recursive: true, force: true }); } + // Failed cleanup remains visible to worker teardown, which preserves builds. + activeProductServerDisposers.delete(dispose); })(); return disposalPromise; }; activeProductServerDisposers.add(dispose); try { + // Register cleanup before configuration, shared preparation or copying can fail. + // Port 0 remains the recipe for every real backend, including restarts. + writeHomeConfig(home, options.auth ? { port: 0 } : { port: 0, auth: false }); + preparation = getSourceCLIEntry(); + sharedEntry = await preparation; + if (disposalRequested) throw new Error("Product server was disposed during preparation"); + if (options.onboardingBundle !== undefined || options.bundledThemesBundle !== undefined) { + bundledRuntime = getCLIEntryWithBundles(sharedEntry, options); + } current = await startServe(); pendingProcess = null; if (disposalRequested) { @@ -246,7 +250,7 @@ export async function launchProductServer(options: LaunchProductServerOptions = rmSync(sibling, { recursive: true, force: true }); cpSync(restartOptions.onboardingBundle, sibling, { recursive: true }); } else { - bundledRuntime = getCLIEntryWithBundles({ + bundledRuntime = getCLIEntryWithBundles(sharedEntry, { onboardingBundle: restartOptions.onboardingBundle, ...(options.bundledThemesBundle !== undefined ? { bundledThemesBundle: options.bundledThemesBundle } diff --git a/test/helpers/versioned-web-bundle.ts b/test/helpers/versioned-web-bundle.ts index 0a926926..083038c1 100644 --- a/test/helpers/versioned-web-bundle.ts +++ b/test/helpers/versioned-web-bundle.ts @@ -1,8 +1,7 @@ -import { mkdtempSync } from "node:fs"; import { createRequire } from "node:module"; -import os from "node:os"; import path from "node:path"; import { pathToFileURL, fileURLToPath } from "node:url"; +import { WorkerBuilds } from "./worker-builds.ts"; // Versioned web-bundle builder for the update-notifications harnesses: a real // `vite build` of packages/web stamped at a chosen version via the sanctioned @@ -15,7 +14,8 @@ import { pathToFileURL, fileURLToPath } from "node:url"; const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); const WEB_PACKAGE_DIR = path.join(REPO_ROOT, "packages", "web"); -const distByVersion = new Map>(); +const builds = new WorkerBuilds("television-web-bundle-"); +export const disposeVersionedWebBuilds = (): Promise => builds.dispose(); // Builds are serialized: the version reaches the vite config through // process.env, which concurrent in-process builds would race on. let buildChain: Promise = Promise.resolve(); @@ -36,9 +36,8 @@ async function loadVite(): Promise { return typeof loaded.build === "function" ? loaded : loaded.default!; } -async function runBuild(version: string | undefined): Promise { +async function runBuild(version: string | undefined, outDir: string): Promise { const vite = await loadVite(); - const outDir = mkdtempSync(path.join(os.tmpdir(), "television-web-bundle-")); const previous = process.env.TV_TEST_WEB_VERSION; if (version === undefined) delete process.env.TV_TEST_WEB_VERSION; else process.env.TV_TEST_WEB_VERSION = version; @@ -52,7 +51,6 @@ async function runBuild(version: string | undefined): Promise { if (previous === undefined) delete process.env.TV_TEST_WEB_VERSION; else process.env.TV_TEST_WEB_VERSION = previous; } - return outDir; } /** @@ -61,11 +59,9 @@ async function runBuild(version: string | undefined): Promise { */ export function buildVersionedWebBundle(version?: string): Promise { const key = version ?? ""; - let dist = distByVersion.get(key); - if (dist === undefined) { - dist = buildChain.then(() => runBuild(version), () => runBuild(version)); - buildChain = dist.catch(() => undefined); - distByVersion.set(key, dist); - } - return dist; + return builds.get(key, (directory) => { + const build = buildChain.then(() => runBuild(version, directory)); + buildChain = build.catch(() => undefined); + return build; + }); } diff --git a/test/helpers/vitest.ts b/test/helpers/vitest.ts new file mode 100644 index 00000000..8adfb808 --- /dev/null +++ b/test/helpers/vitest.ts @@ -0,0 +1,16 @@ +import { test as base } from "vitest"; +import { disposeAllProductServers, disposeSourceCLIBuilds } from "./product-server.ts"; +import { disposeVersionedWebBuilds } from "./versioned-web-bundle.ts"; + +/** Test registration for Vitest callers of the reusable product builders. */ +export const test = base.extend<{ workerBuildCleanup: void }>({ + workerBuildCleanup: [async ({}, use) => { + try { + await use(); + } finally { + await disposeAllProductServers(); + await Promise.all([disposeSourceCLIBuilds(), disposeVersionedWebBuilds()]); + } + }, { scope: "worker", auto: true }], +}); +export const it = test; diff --git a/test/helpers/worker-builds.ts b/test/helpers/worker-builds.ts index c83dcfd9..29ee1deb 100644 --- a/test/helpers/worker-builds.ts +++ b/test/helpers/worker-builds.ts @@ -1,24 +1,45 @@ -import { mkdtempSync } from "node:fs"; +import { mkdtempSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; /** Owns reusable preparation for one runner worker. */ export class WorkerBuilds { private readonly builds = new Map>(); + private readonly directories = new Set(); + private disposal: Promise | undefined; constructor(private readonly prefix: string, private readonly parent = os.tmpdir()) {} get(key: string, prepare: (directory: string) => void | Promise): Promise { + if (this.disposal) return Promise.reject(new Error("Worker build lifetime has ended")); let build = this.builds.get(key); if (!build) { const directory = mkdtempSync(path.join(this.parent, this.prefix)); - build = Promise.resolve().then(() => prepare(directory)).then(() => directory); + this.directories.add(directory); + build = Promise.resolve().then(() => prepare(directory)).then(() => directory, (error: unknown) => { + this.builds.delete(key); + this.remove(directory); + throw error; + }); this.builds.set(key, build); } return build; } - async dispose(): Promise { - await Promise.allSettled(this.builds.values()); + dispose(): Promise { + this.disposal ??= (async () => { + await Promise.allSettled(this.builds.values()); + const failures: unknown[] = []; + for (const directory of this.directories) { + try { this.remove(directory); } catch (error) { failures.push(error); } + } + if (failures.length) throw new AggregateError(failures, "Failed to remove worker builds"); + })(); + return this.disposal; + } + + private remove(directory: string): void { + rmSync(directory, { recursive: true, force: true }); + this.directories.delete(directory); } } diff --git a/test/node/acp-home-context.test.ts b/test/node/acp-home-context.test.ts index a6d18c89..5f7319fb 100644 --- a/test/node/acp-home-context.test.ts +++ b/test/node/acp-home-context.test.ts @@ -2,7 +2,8 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { describe, expect, it } from "vitest"; +import { describe, expect } from "vitest"; +import { it } from "../helpers/vitest.ts"; import { WebSocket } from "ws"; import { ACPClient } from "../../packages/web/src/services/acp-client.ts"; import { launchProductServer } from "../helpers/product-server.ts"; diff --git a/test/node/product-server-proxy.e2e.test.ts b/test/node/product-server-proxy.e2e.test.ts index 7370f107..7187b8da 100644 --- a/test/node/product-server-proxy.e2e.test.ts +++ b/test/node/product-server-proxy.e2e.test.ts @@ -1,6 +1,7 @@ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import net, { type Socket } from "node:net"; -import { afterEach, describe, expect, test } from "vitest"; +import { afterEach, describe, expect } from "vitest"; +import { test } from "../helpers/vitest.ts"; import { disposeAllProductServers, launchProductServer, startDevelopmentProxy, type DevelopmentProxy } from "../helpers/product-server.ts"; import { ownedProcessCount } from "../helpers/owned-process.ts"; @@ -14,11 +15,13 @@ afterEach(async () => { describe("product server lifecycle", () => { test("disposes a child process while product-server startup is still in flight", async () => { - const startup = launchProductServer(); + const startup = launchProductServer({ env: { VITEST: "" } }); + const rejected = expect(startup).rejects.toThrow("tv serve exited before startup"); + await expect.poll(() => ownedProcessCount(), { intervals: [1, 5, 10] }).toBe(1); await disposeAllProductServers(); - await expect(startup).rejects.toThrow("tv serve exited before startup"); + await rejected; expect(ownedProcessCount()).toBe(0); }); }); diff --git a/test/repo/build-config-integrity.test.ts b/test/repo/build-config-integrity.test.ts index 7b740667..aa457832 100644 --- a/test/repo/build-config-integrity.test.ts +++ b/test/repo/build-config-integrity.test.ts @@ -15,7 +15,8 @@ import os from "node:os"; import path from "node:path"; // @ts-expect-error — jsdom is intentionally installed without a declaration package import { JSDOM } from "jsdom"; -import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { afterAll, beforeAll, describe, expect } from "vitest"; +import { test } from "../helpers/vitest.ts"; // @ts-expect-error — repository build script has no declaration file import { buildCanonicalVersions } from "../../packages/canonical/scripts/build-canonical.mjs"; import { buildVersionedWebBundle } from "../helpers/versioned-web-bundle.ts"; From d43623ba6d916a2b161f8161c1a4284767a451d7 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:05:58 +0000 Subject: [PATCH 14/64] Observe built asset loading in the selected product walks --- .../web/test/e2e/browser-demo-mode.test.ts | 15 +++++--- .../test/e2e/theme-product-acceptance.test.ts | 7 ++++ test/helpers/application-requests.ts | 35 +++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 test/helpers/application-requests.ts diff --git a/packages/web/test/e2e/browser-demo-mode.test.ts b/packages/web/test/e2e/browser-demo-mode.test.ts index 47e0e344..0e7b4ba2 100644 --- a/packages/web/test/e2e/browser-demo-mode.test.ts +++ b/packages/web/test/e2e/browser-demo-mode.test.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { test, expect } from "../../../../test/helpers/playwright.ts"; import { launchProductServer, type ProductServer } from "../../../../test/helpers/product-server.ts"; +import { recordApplicationRequests } from "../../../../test/helpers/application-requests.ts"; import { APPLICATION_SHELL_STATES, waitForApplicationRender } from "../../../../test/helpers/application-readiness.ts"; import { configureTestMotion, pickChannelByName } from "./helpers.ts"; @@ -72,11 +73,12 @@ async function createURLArtifact( return ((await response.json()) as { artifact: { id: string } }).artifact.id; } -async function openApp(page: Page, server: ProductServer, baseURL: string | undefined): Promise { +async function openApp(page: Page, server: ProductServer, baseURL: string | undefined, requests: ReturnType): Promise { if (!baseURL) throw new Error("Expected Playwright baseURL"); const appBaseURL = await server.appURL(baseURL); await page.goto(`${appBaseURL}/packages/web/src/index.html`); await waitForApp(page); + requests.expectBuilt(); } async function waitForApp(page: Page): Promise { @@ -106,8 +108,10 @@ test.describe("browser demo mode", () => { let home: string; let site: Awaited>; let server: ProductServer | null = null; + let requests: ReturnType; - test.beforeEach(async () => { + test.beforeEach(async ({ page }) => { + requests = recordApplicationRequests([page]); home = mkdtempSync(path.join(os.tmpdir(), "television-demo-home-")); site = await startExternalSite(); server = await launchProductServer({ @@ -117,6 +121,7 @@ test.describe("browser demo mode", () => { }); test.afterEach(async () => { + await requests.attach(test.info()); await server?.dispose(); server = null; await site.close(); @@ -132,7 +137,7 @@ test.describe("browser demo mode", () => { await createURLArtifact(server, pageURL, channel.id); const returnButton = page.getByRole("button", { name: RETURN_LABEL }); - await openApp(page, server, baseURL); + await openApp(page, server, baseURL, requests); await expect(artifactFrame(page)).toHaveAttribute("src", /\/views\/url-unsupported\//); await expect(returnButton).toHaveCount(0); @@ -160,7 +165,7 @@ test.describe("browser demo mode", () => { await createChannel(server, "Elsewhere"); await createURLArtifact(server, `${site.origin}/one`, channel.id); - await openApp(page, server, baseURL); + await openApp(page, server, baseURL, requests); await expect(frameHeading(page)).toHaveText("Fixture page one"); await followNext(page); @@ -191,7 +196,7 @@ test.describe("browser demo mode", () => { await createURLArtifact(server, pageURL, channel.id); await createURLArtifact(server, `${site.origin}/two`, channel.id, "Second page"); - await openApp(page, server, baseURL); + await openApp(page, server, baseURL, requests); const selected = page.getByRole("option", { name: channel.name, exact: true }); await expect(selected).toHaveAttribute("aria-selected", "true"); const tabs = page.locator(".top-bar .tab"); diff --git a/packages/web/test/e2e/theme-product-acceptance.test.ts b/packages/web/test/e2e/theme-product-acceptance.test.ts index f3180668..b0727e68 100644 --- a/packages/web/test/e2e/theme-product-acceptance.test.ts +++ b/packages/web/test/e2e/theme-product-acceptance.test.ts @@ -13,6 +13,7 @@ import { TelevisionClient } from "@telepath-computer/television-shared"; import { test, expect } from "../../../../test/helpers/playwright.ts"; import { launchProductServer, type ProductServer } from "../../../../test/helpers/product-server.ts"; import { seedThemePackage } from "../../../../test/helpers/theme-package.ts"; +import { recordApplicationRequests } from "../../../../test/helpers/application-requests.ts"; import { waitForApplicationShell } from "./helpers.ts"; import { APPLICATION_SHELL_STATES, @@ -616,6 +617,7 @@ test("theme settings persist rendered app and artifact outcomes through a built- if (!baseURL) throw new Error("Expected Playwright baseURL"); const context = await browser.newContext({ colorScheme: null }); const page = await context.newPage(); + const requests = recordApplicationRequests([page]); const product = await launchProductServer(); try { const themeAPath = seedThemePackage( @@ -694,6 +696,7 @@ test("theme settings persist rendered app and artifact outcomes through a built- const appURL = await product.appURL(baseURL); await page.goto(`${appURL}/packages/web/src/index.html`); await waitForApplicationShell(page); + requests.expectBuilt(); await openSettings(page); await page.getByRole("button", { name: "Refresh themes" }).click(); @@ -868,6 +871,7 @@ test("theme settings persist rendered app and artifact outcomes through a built- expect((await expectSandboxFrame(page, "overlay")).instance) .not.toBe(beforeAdaptiveOverlay.instance); } finally { + await requests.attach(test.info()); await product.dispose(); await context.close(); } @@ -895,6 +899,7 @@ test("executable themes run only in the Chromium application lifecycle and isola const product = await launchProductServer(); const secondPage = await page.context().newPage(); const applicationPages = [page, secondPage] as const; + const requests = recordApplicationRequests(applicationPages); const selectReloadingTheme = async (value: string): Promise => { // A consented script is cleared by replacing both application documents. // Observe that navigation before applying ordinary post-render assertions. @@ -1069,6 +1074,7 @@ document.querySelector("#artifact-probe").addEventListener("click", () => { ); await waitForApplicationShell(applicationPage); })); + requests.expectBuilt(); await openSettings(page); await page.getByRole("button", { name: "Refresh themes" }).click(); const themeSelect = page.locator('tv-select[name="theme"]'); @@ -1845,6 +1851,7 @@ document.querySelector("#artifact-probe").addEventListener("click", () => { expect(scriptRequests.length).toBeGreaterThan(0); expect(scriptRequests.every((owner) => owner === "application")).toBe(true); } finally { + await requests.attach(test.info()); await product.dispose(); await secondPage.close(); } diff --git a/test/helpers/application-requests.ts b/test/helpers/application-requests.ts new file mode 100644 index 00000000..08fad8d5 --- /dev/null +++ b/test/helpers/application-requests.ts @@ -0,0 +1,35 @@ +import { expect, type Page, type TestInfo } from "@playwright/test"; + +/** Retain paths only: authentication fragments and query values are never recorded. */ +export function recordApplicationRequests(pages: readonly Page[]) { + const inventories = pages.map((page) => { + const entries = new Set(); + page.on("request", (request) => { + entries.add(JSON.stringify({ + path: new URL(request.url()).pathname, + type: request.resourceType(), + application: request.frame() === page.mainFrame(), + })); + }); + return entries; + }); + const snapshot = () => inventories.map((entries) => [...entries].sort().map((entry) => + JSON.parse(entry) as { path: string; type: string; application: boolean } + )); + return { + expectBuilt(): void { + for (const requests of snapshot()) { + const application = requests.filter((request) => request.application); + expect(application.some((request) => /^\/(packages|@vite|@id|@fs|node_modules)\//.test(request.path)), + "the application loads shipped assets without development modules").toBe(false); + expect(application.some((request) => request.type === "document" && request.path === "/")).toBe(true); + expect(application.some((request) => request.type === "script" && /^\/assets\/.+\.js$/.test(request.path))).toBe(true); + } + }, + async attach(info: TestInfo): Promise { + await info.attach("application-request-inventory", { + body: Buffer.from(JSON.stringify(snapshot(), null, 2)), contentType: "application/json", + }); + }, + }; +} From a2fd5a1e3c608d39c3f6a4f67ee507f91289d7d8 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:08:17 +0000 Subject: [PATCH 15/64] Use built assets in five product walks and close remaining build lifetimes --- .../web/test/e2e/browser-demo-mode.test.ts | 18 ++++++++---------- .../web/test/e2e/gate-boot-barrier.spec.ts | 3 ++- .../test/e2e/theme-product-acceptance.test.ts | 19 +++++++++++-------- packages/web/test/e2e/update-journey.spec.ts | 3 ++- .../test/e2e/update-notification.01.spec.ts | 3 ++- .../test/e2e/update-notification.02.spec.ts | 3 ++- packages/web/test/e2e/update-reload.spec.ts | 3 ++- test/helpers/product-server.ts | 5 +++++ test/node/update-reload-guard.e2e.test.ts | 3 ++- test/node/update-reload.e2e.test.ts | 3 ++- 10 files changed, 38 insertions(+), 25 deletions(-) diff --git a/packages/web/test/e2e/browser-demo-mode.test.ts b/packages/web/test/e2e/browser-demo-mode.test.ts index 0e7b4ba2..520e6006 100644 --- a/packages/web/test/e2e/browser-demo-mode.test.ts +++ b/packages/web/test/e2e/browser-demo-mode.test.ts @@ -73,10 +73,8 @@ async function createURLArtifact( return ((await response.json()) as { artifact: { id: string } }).artifact.id; } -async function openApp(page: Page, server: ProductServer, baseURL: string | undefined, requests: ReturnType): Promise { - if (!baseURL) throw new Error("Expected Playwright baseURL"); - const appBaseURL = await server.appURL(baseURL); - await page.goto(`${appBaseURL}/packages/web/src/index.html`); +async function openApp(page: Page, server: ProductServer, requests: ReturnType): Promise { + await page.goto(server.serverURL); await waitForApp(page); requests.expectBuilt(); } @@ -129,7 +127,7 @@ test.describe("browser demo mode", () => { }); // spec: proofs/product/artifacts.md#^af-ac-demo-mode - test("the home-directory marker turns direct external pages on and off at app load (^af-ac-demo-mode)", async ({ page, baseURL }) => { + test("the home-directory marker turns direct external pages on and off at app load (^af-ac-demo-mode)", async ({ page }) => { if (!server) throw new Error("Product server was not started"); const [channel] = await channels(server); if (!channel) throw new Error("Expected a default channel"); @@ -137,7 +135,7 @@ test.describe("browser demo mode", () => { await createURLArtifact(server, pageURL, channel.id); const returnButton = page.getByRole("button", { name: RETURN_LABEL }); - await openApp(page, server, baseURL, requests); + await openApp(page, server, requests); await expect(artifactFrame(page)).toHaveAttribute("src", /\/views\/url-unsupported\//); await expect(returnButton).toHaveCount(0); @@ -157,7 +155,7 @@ test.describe("browser demo mode", () => { }); // spec: proofs/product/artifacts.md#^af-ac-demo-return - test("the artifact icon, an app reload, and a channel round trip each return to the original page (^af-ac-demo-return)", async ({ page, baseURL }) => { + test("the artifact icon, an app reload, and a channel round trip each return to the original page (^af-ac-demo-return)", async ({ page }) => { if (!server) throw new Error("Product server was not started"); writeFileSync(path.join(home, MARKER), ""); const [channel] = await channels(server); @@ -165,7 +163,7 @@ test.describe("browser demo mode", () => { await createChannel(server, "Elsewhere"); await createURLArtifact(server, `${site.origin}/one`, channel.id); - await openApp(page, server, baseURL, requests); + await openApp(page, server, requests); await expect(frameHeading(page)).toHaveText("Fixture page one"); await followNext(page); @@ -186,7 +184,7 @@ test.describe("browser demo mode", () => { }); // spec: proofs/arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-seam - test("bridge-shaped messages from a demo-mode page change nothing (^ab-ac-demo-inert-seam)", async ({ page, baseURL }) => { + test("bridge-shaped messages from a demo-mode page change nothing (^ab-ac-demo-inert-seam)", async ({ page }) => { if (!server) throw new Error("Product server was not started"); writeFileSync(path.join(home, MARKER), ""); const [channel] = await channels(server); @@ -196,7 +194,7 @@ test.describe("browser demo mode", () => { await createURLArtifact(server, pageURL, channel.id); await createURLArtifact(server, `${site.origin}/two`, channel.id, "Second page"); - await openApp(page, server, baseURL, requests); + await openApp(page, server, requests); const selected = page.getByRole("option", { name: channel.name, exact: true }); await expect(selected).toHaveAttribute("aria-selected", "true"); const tabs = page.locator(".top-bar .tab"); diff --git a/packages/web/test/e2e/gate-boot-barrier.spec.ts b/packages/web/test/e2e/gate-boot-barrier.spec.ts index 6759b9b8..82dacf48 100644 --- a/packages/web/test/e2e/gate-boot-barrier.spec.ts +++ b/packages/web/test/e2e/gate-boot-barrier.spec.ts @@ -1,4 +1,5 @@ -import { expect, test, type Page } from "@playwright/test"; +import { type Page } from "@playwright/test"; +import { expect, test } from "../../../../test/helpers/playwright.ts"; import { mkdtempSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; diff --git a/packages/web/test/e2e/theme-product-acceptance.test.ts b/packages/web/test/e2e/theme-product-acceptance.test.ts index b0727e68..071fffe8 100644 --- a/packages/web/test/e2e/theme-product-acceptance.test.ts +++ b/packages/web/test/e2e/theme-product-acceptance.test.ts @@ -14,6 +14,7 @@ import { test, expect } from "../../../../test/helpers/playwright.ts"; import { launchProductServer, type ProductServer } from "../../../../test/helpers/product-server.ts"; import { seedThemePackage } from "../../../../test/helpers/theme-package.ts"; import { recordApplicationRequests } from "../../../../test/helpers/application-requests.ts"; +import { startStableFrontProxy } from "../../../../test/helpers/stable-front-proxy.ts"; import { waitForApplicationShell } from "./helpers.ts"; import { APPLICATION_SHELL_STATES, @@ -612,13 +613,12 @@ test("Firefox refreshes a ready Markdown editor from a fixed-dark theme to light // proofs/arch/themes/delivery.md#^theme-delivery-t-active-package-refresh test("theme settings persist rendered app and artifact outcomes through a built-server restart", async ({ browser, - baseURL, }) => { - if (!baseURL) throw new Error("Expected Playwright baseURL"); const context = await browser.newContext({ colorScheme: null }); const page = await context.newPage(); const requests = recordApplicationRequests([page]); const product = await launchProductServer(); + const front = await startStableFrontProxy(product.serverURL); try { const themeAPath = seedThemePackage( product.home, @@ -693,8 +693,8 @@ test("theme settings persist rendered app and artifact outcomes through a built- "https://example.com/appearance", ); - const appURL = await product.appURL(baseURL); - await page.goto(`${appURL}/packages/web/src/index.html`); + const appURL = front.url; + await page.goto(appURL); await waitForApplicationShell(page); requests.expectBuilt(); @@ -846,6 +846,7 @@ test("theme settings persist rendered app and artifact outcomes through a built- )).toHaveCount(0); await product.restart(); + front.setTarget(product.serverURL); await waitForApplicationShell(page); await expectAppearance("dark", FIXED_DARK); const persistedDisplay = await fetch(new URL("/display", product.serverURL)); @@ -872,6 +873,7 @@ test("theme settings persist rendered app and artifact outcomes through a built- .not.toBe(beforeAdaptiveOverlay.instance); } finally { await requests.attach(test.info()); + await front.dispose(); await product.dispose(); await context.close(); } @@ -884,7 +886,6 @@ test("theme settings persist rendered app and artifact outcomes through a built- // proofs/arch/themes/delivery.md#^theme-delivery-t-frame-appearance-browser test("executable themes run only in the Chromium application lifecycle and isolate failures", async ({ page, - baseURL, browserName, }) => { test.skip(browserName !== "chromium", "Chromium owns executable-theme browser acceptance"); @@ -894,9 +895,9 @@ test("executable themes run only in the Chromium application lifecycle and isola const reconnectBackoffBudget = 30_000; const themeResetNavigationBudget = 30_000; test.setTimeout(test.info().timeout + reconnectBackoffBudget + 2 * themeResetNavigationBudget); - if (!baseURL) throw new Error("Expected Playwright baseURL"); const product = await launchProductServer(); + const front = await startStableFrontProxy(product.serverURL); const secondPage = await page.context().newPage(); const applicationPages = [page, secondPage] as const; const requests = recordApplicationRequests(applicationPages); @@ -1067,10 +1068,10 @@ document.querySelector("#artifact-probe").addEventListener("click", () => { "Executable Theme Artifact", artifactPath, ); - const appURL = await product.appURL(baseURL); + const appURL = front.url; await Promise.all(applicationPages.map(async (applicationPage) => { await applicationPage.goto( - `${appURL}/packages/web/src/index.html`, + appURL, ); await waitForApplicationShell(applicationPage); })); @@ -1789,6 +1790,7 @@ document.querySelector("#artifact-probe").addEventListener("click", () => { await expectExecutableRuns(applicationPages, 3); await product.restart(); + front.setTarget(product.serverURL); // Allow one capped retry interval plus the usual 5s shell-ready budget. await Promise.all(applicationPages.map((applicationPage) => waitForApplicationRender( @@ -1852,6 +1854,7 @@ document.querySelector("#artifact-probe").addEventListener("click", () => { expect(scriptRequests.every((owner) => owner === "application")).toBe(true); } finally { await requests.attach(test.info()); + await front.dispose(); await product.dispose(); await secondPage.close(); } diff --git a/packages/web/test/e2e/update-journey.spec.ts b/packages/web/test/e2e/update-journey.spec.ts index 42db665c..5ad0664c 100644 --- a/packages/web/test/e2e/update-journey.spec.ts +++ b/packages/web/test/e2e/update-journey.spec.ts @@ -1,4 +1,5 @@ -import { expect, test, type Page } from "@playwright/test"; +import { type Page } from "@playwright/test"; +import { expect, test } from "../../../../test/helpers/playwright.ts"; import { mkdtempSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; diff --git a/packages/web/test/e2e/update-notification.01.spec.ts b/packages/web/test/e2e/update-notification.01.spec.ts index 2efacd40..f7841cc8 100644 --- a/packages/web/test/e2e/update-notification.01.spec.ts +++ b/packages/web/test/e2e/update-notification.01.spec.ts @@ -1,4 +1,5 @@ -import { expect, test, type Locator, type Page } from "@playwright/test"; +import { type Locator, type Page } from "@playwright/test"; +import { expect, test } from "../../../../test/helpers/playwright.ts"; import { mkdtempSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; diff --git a/packages/web/test/e2e/update-notification.02.spec.ts b/packages/web/test/e2e/update-notification.02.spec.ts index a5cbbc2c..32f84770 100644 --- a/packages/web/test/e2e/update-notification.02.spec.ts +++ b/packages/web/test/e2e/update-notification.02.spec.ts @@ -1,4 +1,5 @@ -import { expect, test, type Page } from "@playwright/test"; +import { type Page } from "@playwright/test"; +import { expect, test } from "../../../../test/helpers/playwright.ts"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; diff --git a/packages/web/test/e2e/update-reload.spec.ts b/packages/web/test/e2e/update-reload.spec.ts index 8df87161..63589ffc 100644 --- a/packages/web/test/e2e/update-reload.spec.ts +++ b/packages/web/test/e2e/update-reload.spec.ts @@ -1,4 +1,5 @@ -import { expect, test, type Page } from "@playwright/test"; +import { type Page } from "@playwright/test"; +import { expect, test } from "../../../../test/helpers/playwright.ts"; import { mkdtempSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; diff --git a/test/helpers/product-server.ts b/test/helpers/product-server.ts index db6902a3..e9d0ff29 100644 --- a/test/helpers/product-server.ts +++ b/test/helpers/product-server.ts @@ -108,6 +108,7 @@ function getCLIEntryWithBundles(sharedEntry: string, options: { function installProductRuntimeAssets(buildDir: string): void { const webDist = path.join(REPO_ROOT, "packages/web/dist"); const artifactMissingView = path.join(webDist, "views/artifact-missing"); + const urlUnsupportedView = path.join(webDist, "views/url-unsupported"); const markdownView = path.join(REPO_ROOT, "packages/view-markdown/dist"); const canonical = path.join(REPO_ROOT, "packages/canonical/dist/canonical"); const sdk = path.join(REPO_ROOT, "packages/server/dist/sdk"); @@ -116,6 +117,9 @@ function installProductRuntimeAssets(buildDir: string): void { if (!existsSync(path.join(artifactMissingView, "index.html"))) { throw new Error(`Web build produced no artifact-missing view at ${artifactMissingView}`); } + if (!existsSync(path.join(urlUnsupportedView, "index.html"))) { + throw new Error(`Web build produced no url-unsupported view at ${urlUnsupportedView}`); + } if (!existsSync(path.join(markdownView, "index.html"))) { throw new Error(`View build produced no markdown view at ${markdownView}`); } @@ -133,6 +137,7 @@ function installProductRuntimeAssets(buildDir: string): void { } cpSync(artifactMissingView, path.join(buildDir, "views/artifact-missing"), { recursive: true }); + cpSync(urlUnsupportedView, path.join(buildDir, "views/url-unsupported"), { recursive: true }); cpSync(markdownView, path.join(buildDir, "views/markdown"), { recursive: true }); cpSync(canonical, path.join(buildDir, "canonical"), { recursive: true }); cpSync(sdk, path.join(buildDir, "sdk"), { recursive: true }); diff --git a/test/node/update-reload-guard.e2e.test.ts b/test/node/update-reload-guard.e2e.test.ts index 1ff46e11..567a3183 100644 --- a/test/node/update-reload-guard.e2e.test.ts +++ b/test/node/update-reload-guard.e2e.test.ts @@ -1,4 +1,5 @@ -import { describe, it } from "vitest"; +import { describe } from "vitest"; +import { it } from "../helpers/vitest.ts"; import { expect as pwExpect } from "@playwright/test"; import { buildVersionedWebBundle, diff --git a/test/node/update-reload.e2e.test.ts b/test/node/update-reload.e2e.test.ts index 1f359bbe..b8c68916 100644 --- a/test/node/update-reload.e2e.test.ts +++ b/test/node/update-reload.e2e.test.ts @@ -1,4 +1,5 @@ -import { describe, it } from "vitest"; +import { describe } from "vitest"; +import { it } from "../helpers/vitest.ts"; import { expect as pwExpect } from "@playwright/test"; import { buildVersionedWebBundle, From 3156e3031070c3adfdc3ed780b9bff6817eafd78 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:10:40 +0000 Subject: [PATCH 16/64] Place the unsupported URL document under the static web root --- test/helpers/product-server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/helpers/product-server.ts b/test/helpers/product-server.ts index e9d0ff29..a8fc6eb3 100644 --- a/test/helpers/product-server.ts +++ b/test/helpers/product-server.ts @@ -137,12 +137,12 @@ function installProductRuntimeAssets(buildDir: string): void { } cpSync(artifactMissingView, path.join(buildDir, "views/artifact-missing"), { recursive: true }); - cpSync(urlUnsupportedView, path.join(buildDir, "views/url-unsupported"), { recursive: true }); cpSync(markdownView, path.join(buildDir, "views/markdown"), { recursive: true }); cpSync(canonical, path.join(buildDir, "canonical"), { recursive: true }); cpSync(sdk, path.join(buildDir, "sdk"), { recursive: true }); const staticDir = path.join(buildDir, "web"); mkdirSync(staticDir, { recursive: true }); + cpSync(urlUnsupportedView, path.join(staticDir, "views/url-unsupported"), { recursive: true }); cpSync(webIndex, path.join(staticDir, "index.html")); cpSync(webAssets, path.join(staticDir, "assets"), { recursive: true }); } From 502ba2555cc7a761611e7059800e6903cc74c205 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:16:53 +0000 Subject: [PATCH 17/64] Keep worker build helpers compatible with the repository TypeScript target --- test/helpers/worker-builds.ts | 7 ++++++- test/repo/worker-builds.test.ts | 11 ++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/test/helpers/worker-builds.ts b/test/helpers/worker-builds.ts index 29ee1deb..d6c7a6e7 100644 --- a/test/helpers/worker-builds.ts +++ b/test/helpers/worker-builds.ts @@ -7,8 +7,13 @@ export class WorkerBuilds { private readonly builds = new Map>(); private readonly directories = new Set(); private disposal: Promise | undefined; + private readonly prefix: string; + private readonly parent: string; - constructor(private readonly prefix: string, private readonly parent = os.tmpdir()) {} + constructor(prefix: string, parent = os.tmpdir()) { + this.prefix = prefix; + this.parent = parent; + } get(key: string, prepare: (directory: string) => void | Promise): Promise { if (this.disposal) return Promise.reject(new Error("Worker build lifetime has ended")); diff --git a/test/repo/worker-builds.test.ts b/test/repo/worker-builds.test.ts index 8a127b26..9630b5cb 100644 --- a/test/repo/worker-builds.test.ts +++ b/test/repo/worker-builds.test.ts @@ -5,6 +5,11 @@ import { afterEach, expect, test } from "vitest"; import { WorkerBuilds } from "../helpers/worker-builds.ts"; const roots: string[] = []; +function rendezvous() { + let resolve!: (value: T) => void; + const promise = new Promise((done) => { resolve = done; }); + return { promise, resolve }; +} function fixture(): { root: string; builds: WorkerBuilds } { const root = mkdtempSync(path.join(os.tmpdir(), "worker-build-contract-")); roots.push(root); @@ -15,7 +20,7 @@ afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: // proofs/arch/testing-policy.md#^testing-t-build-lifetime test("concurrent and completed requests reuse a build while version keys remain distinct", async () => { const { root, builds } = fixture(); - const ready = Promise.withResolvers(); + const ready = rendezvous(); let calls = 0; const first = builds.get("v1", async (directory) => { calls += 1; @@ -52,8 +57,8 @@ test("failed preparation removes partial output and permits a fresh build", asyn test.each([false, true])("teardown waits for pending preparation, including rejection (%s)", async (reject) => { const { root, builds } = fixture(); - const started = Promise.withResolvers(); - const ready = Promise.withResolvers(); + const started = rendezvous(); + const ready = rendezvous(); const build = builds.get("pending", async (directory) => { writeFileSync(path.join(directory, "partial.js"), "pending"); started.resolve(directory); From 389764557e1714af1d6c0009dcb9f9c01aca19ed Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:19:09 +0000 Subject: [PATCH 18/64] Use the Vitest polling interval option for startup disposal --- test/node/product-server-proxy.e2e.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/node/product-server-proxy.e2e.test.ts b/test/node/product-server-proxy.e2e.test.ts index 7187b8da..50583715 100644 --- a/test/node/product-server-proxy.e2e.test.ts +++ b/test/node/product-server-proxy.e2e.test.ts @@ -17,7 +17,7 @@ describe("product server lifecycle", () => { test("disposes a child process while product-server startup is still in flight", async () => { const startup = launchProductServer({ env: { VITEST: "" } }); const rejected = expect(startup).rejects.toThrow("tv serve exited before startup"); - await expect.poll(() => ownedProcessCount(), { intervals: [1, 5, 10] }).toBe(1); + await expect.poll(() => ownedProcessCount(), { interval: 5 }).toBe(1); await disposeAllProductServers(); From 036d04910384ea0f6d8aa4cb4ce55cc8067f4d15 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:27:37 +0000 Subject: [PATCH 19/64] Record slice one worker lifetime and built-app validation --- .../test-harness-resource-cleanup-plan.md | 4 +- ...arness-resource-cleanup-review-response.md | 14 +++- .../test-harness-resource-cleanup-slice-1.md | 66 +++++++++++++++++++ proofs/arch/artifact-frame/artifact-bridge.md | 4 +- proofs/arch/testing-policy.md | 6 +- proofs/arch/themes/delivery.md | 8 +-- proofs/product/artifacts.md | 6 +- proofs/product/themes-and-appearance.md | 14 ++-- 8 files changed, 100 insertions(+), 22 deletions(-) create mode 100644 docs/working/test-harness-resource-cleanup-slice-1.md diff --git a/docs/working/test-harness-resource-cleanup-plan.md b/docs/working/test-harness-resource-cleanup-plan.md index 59c9b580..92c0c68d 100644 --- a/docs/working/test-harness-resource-cleanup-plan.md +++ b/docs/working/test-harness-resource-cleanup-plan.md @@ -1,6 +1,6 @@ # Test-harness resource cleanup: implementation plan -Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; slice 1 implementation is underway. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. +Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; [slice 1 is implemented](test-harness-resource-cleanup-slice-1.md) for independent review. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. ## Slice judgment @@ -93,4 +93,4 @@ Retrieve all evidence before deleting the dedicated coordinator, and report clea ## Current limits -No new tests or implementation exist at this checkpoint and no test or worker execution has occurred. The coordinator's local preflight capabilities, credentials and capacity; actual provider behavior during renewal, label changes and interrupted dispatch; browser temporary placement; and the implemented tree's full-gate duration remain unmeasured. The plan carries those checks into the slices that can establish them. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. +Slice 1's helper contracts, real framework teardown and five built-application walks have red/green evidence, with affected-file and caller checks recorded in its evidence document. The named private coordinator and two workers exist; credentials, Node and x64/Linux execution are established there. Provider inventory differs from the historical pool count, and account-specific quota and idle-sandbox accounting remain unknown. The coordinator's experiment-driver preflight, renewal/recovery and pool-label behavior, interrupted dispatch, browser temporary placement, diagnostic transport and full-gate duration remain for slices 2 and 3. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 414baa49..2b968b1e 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -1,6 +1,6 @@ # Resource-cleanup proposal: review response -Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, proof round 2 at `3fa15d47`, and plan round 2 at `04a6b275` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. Slice 1 implementation is underway. Earlier sections retain the response history, including the superseded private full-gate arrangement. Reviewer: Claude Fable (Nina), arranged by the supervisor. +Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, proof round 2 at `3fa15d47`, and plan round 2 at `04a6b275` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. Slice 1 is implemented for independent review; its evidence is linked below. Earlier sections retain the response history, including the superseded private full-gate arrangement. Reviewer: Claude Fable (Nina), arranged by the supervisor. Independent round 1 reviewed `1652787e`. The [full review](test-harness-resource-cleanup-review-r1.md) remains unchanged: FAIL on two blockers. The response committed at `99b7ed93` narrowed the work pending Josh's decisions. The following sections through round 2 record that history, not the current review verdict. @@ -198,3 +198,15 @@ The [complete PASS report](test-harness-resource-cleanup-plan-review-r2.md) is p Read-only provider inventory found 143 deployed `poc` sandboxes and none of the named private hosts. The documented 72-worker inventory is therefore not current. Blaxel accepted creation of `tv-cleanup-coordinator-rose`; the canonical pool manager, running there at pushed `04a6b275`, provisioned `tv-testshard-x64-rose-cleanup-01` and `-02`. Each uses the named Playwright image, 8192 MB and Node 24.22.1/npm 11.19.1, with x64/Linux verified on the workers. API-key authentication and the GitHub credential are available; values were passed only through the SDK process environment. The provider accepted this capacity, but its account-specific quota and idle-sandbox accounting remain unknown. No shared worker was modified. The first red checkpoint adds a real Playwright/Vitest worker fixture and parent seam. Each worker uses real source CLI and versioned web output in two cases, observes reuse after per-case cleanup, and leaves its outputs for the parent to inspect after normal teardown. An absent bundle exercises failed-launch private-copy cleanup; a later worker must receive different paths. Nested fixture retries are zero to observe exactly two cases; the canonical parent retains its default retry budget. Test execution has not yet occurred at this checkpoint. All subsequent commands and evidence are recorded below. + +## Slice 1: implemented for independent review + +The [slice 1 evidence record](test-harness-resource-cleanup-slice-1.md) contains the red/green revisions, exact selections, canonical run directories, request inventories, failed intermediate attempts and remaining limits. The code checkpoint is `38976455`; the following proof/document update changes no executable behavior. + +The shared build owner now publishes only successful preparation, reuses each worker's source CLI and each versioned web build, removes failed output and waits for preparation during teardown. Real Playwright and Vitest worker fixtures close that lifetime after server cleanup. Per-launch cleanup is registered before preparation and removes failed private copies and homes. Existing direct and indirect builder callers receive the appropriate fixture registration. The selected two theme and three demo-mode cases now load built assets while retaining their product assertions; PR #9's reset and the separate Settings development walk remain unchanged. The theme restart path uses the existing stable-front proxy, and the helper supplies the built unsupported-URL view at the server's static path. + +Four contract cases and the real two-framework seam have intended red/green evidence. The five converted walks first failed on their observed development-module requests, then passed on built application requests. Both affected browser files and ten focused caller selections passed, as did remote lint/type checking and the final cache/startup-disposal checks. Completed green runs needed no recovered retries and reported no process leaks. The final proxy check had one separate incomplete attempt when the coordinator's Git fetch stalled before acquisition; the confirmed fetch child was stopped, its failure retained, and the unchanged selection passed on repetition. The evidence record distinguishes that infrastructure interruption from test failures. + +All execution occurred on the named private Blaxel hosts at pushed revisions; nothing ran on the shared development host and no shared worker was modified. The five mirrored proof files now cite the implemented, passing coverage. The optional Firefox project was unavailable; no new Firefox validation is claimed. Slice 1's killed-worker backstop, diagnostic transport, renewable worker ownership and recovery remain assigned to slices 2 and 3. Full verification and integrated review are still pending. The private hosts remain allocated for those slices. + +The supervisor arranges independent review of this slice before further implementation. No new durable team decision was made; this work implements the converged proposal, specs, proofs and plan. diff --git a/docs/working/test-harness-resource-cleanup-slice-1.md b/docs/working/test-harness-resource-cleanup-slice-1.md new file mode 100644 index 00000000..b6ddc766 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-slice-1.md @@ -0,0 +1,66 @@ +# Test-harness cleanup: slice 1 evidence + +Owner: rose. Draft PR #12, branch `thopter/test-harness-resource-cleanup`. Derived from the converged [plan](test-harness-resource-cleanup-plan.md). This record covers worker-owned builds and the five built-application walks. It does not claim the diagnostics or remote worker lifecycle from slices 2 and 3. + +## Changes and review scope + +`WorkerBuilds` owns temporary directories before preparation starts, shares pending and successful results by key, removes failed output and permits retry, and waits for preparation before worker teardown removes the remaining output. Both real builders use it. The web builder retains serialized environment overrides and version-keyed reuse. Source CLI preparation now returns its entry only after the build and runtime assets succeed. + +Playwright's existing shared registration now has a worker fixture in addition to its per-case server cleanup. Vitest callers use a small worker-scoped fixture registration. Normal server disposal preserves reusable builds. Worker cleanup stops registered servers before removing builds; failed server cleanup remains registered and prevents build removal. Per-launch cleanup is registered before configuration or preparation, and a failed private bundle copy removes both that runtime and its launch-owned home. + +The five selected walks load built assets: persistence and executable lifecycle in `theme-product-acceptance.test.ts`, plus all three demo-mode cases. The theme walks keep their browser origins through the existing stable-front proxy while each replacement backend binds port `0`. Existing product, appearance, motion and authentication behavior is preserved. The unsupported-URL document is copied under the test runtime's `web/views/url-unsupported` static path, matching the production package layout. PR #9's reset test and the separate Settings development-module walk are unchanged. + +Five already-built browser update/gate files and two Node update/reload files also change their test registration so their versioned web outputs receive teardown. Their loading and assertions do not change. The other Vitest registrations are the source-CLI proxy and ACP callers, foundation-copy tests, and build-config integrity. No production application file, dependency, timeout default or retry default changes. + +## Placement and red/green + +Every test ran from `/workspace/television` on `tv-cleanup-coordinator-rose`, using the canonical Blaxel runner and private pool `rose-cleanup`. Native execution ran on `tv-testshard-x64-rose-cleanup-01` or `-02`, at committed, pushed revisions. No tests, builds, lint, type checks, browsers or servers ran on the shared development host. The shared pool was not used or modified. Default canonical retries were retained: two for e2e and zero for unit surfaces, plus two infrastructure retries. The nested worker fixtures deliberately use zero retries so each child records exactly two cases; their canonical parent retains the e2e budget. + +- `564a8edb` stopped at registry validation before execution because two static nested configs looked like unregistered surfaces. This is setup failure, not red evidence. `74485d22` generates the native configs inside the acceptance's private temporary directory, following the existing nested-runner arrangement. +- At `74485d22`, the Playwright child successfully built and used the real output in both cases, but the parent found four retained harness directories after worker exit. The Vitest case established the same failure at `2460d06a`. All three canonical attempts failed on that assertion, with no process-leak or infrastructure failure. +- The four cache contracts at `2460d06a` failed on retained partial/completed output. Their builders are the proof-declared substitutes and use a real temporary filesystem. The lifecycle seam, rather than these callbacks, establishes usable real CLI/web output and actual framework teardown. +- At `8bfd952f`, all four contracts, both framework seam cases, and the existing startup-disposal/proxy file passed. Each framework ran two worker lifetimes; cases reused output within a worker, later workers had separate paths, failed-copy output disappeared before the valid launch, and teardown left no owned build or launch directories. +- At `d43623ba`, all five selected browser cases reached and failed the new request-boundary assertion: development modules were still requested. At `a2fd5a1e`, the executable walk passed but persistence failed on a 404 for the unsupported-URL view. The helper had copied it to the sibling-view location even though the server serves this document through the web static root. `3156e303` corrects only that destination. The narrow persistence run, whole demo file and whole theme file then passed. +- Inspection of the repository TypeScript target prompted `502ba255`: explicit constructor field assignments replace parameter properties, and test rendezvous promises use ES2022-compatible construction. This changes no lifetime behavior. Remote lint passed at that revision; type checking found that the adapted startup-disposal assertion had used Playwright’s plural `intervals` instead of Vitest’s singular `interval`. `38976455` corrects the option. Both `npm run lint` and `npm run type-check` then passed on the private coordinator at that pushed revision, and the cache-contract and startup-disposal files passed there through the canonical Blaxel runner. Static logs are retained locally under `.test-runs/slice-1-static/`. + +The run table below records the native evidence and exact canonical selections. Each command has the prefix `npm test -- blaxel --pool rose-cleanup`; the `--run-dir-output` receipt used for orchestration changes no selection. Every run directory is under `/workspace/television/.test-runs/` on the coordinator and has also been copied under `.test-runs/` in the contribution worktree. `summary.json`, `results.json`, `events.ndjson`, the provider request/report/setup/shard files and native JSON are retained for executed runs. The interrupted fetch has only canonical summaries/events and its coordinator log; it never produced a provider or native report. The native JSON embeds request inventories, so they remain available after later worker checkout or cleanup. + +| Commit | Canonical selection after the command prefix | Result | Run-directory name | +| --- | --- | --- | --- | +| `74485d22` | `--file test/node/worker-build-lifetime.e2e.test.ts --grep 'playwright workers'` | failed (1 failing case) | `2026-10-27T16-59-11-096Z-p2995-r90394ebd75419cf9` | +| `2460d06a` | `--file test/repo/worker-builds.test.ts` | failed (4 failing cases) | `2026-10-27T17-01-33-418Z-p3544-rf0ed6ae6891b7bef` | +| `2460d06a` | `--file test/node/worker-build-lifetime.e2e.test.ts --grep 'vitest workers'` | failed (1 failing case) | `2026-10-27T17-02-51-359Z-p4091-r5570749d7be150b1` | +| `8bfd952f` | `--file test/repo/worker-builds.test.ts` | passed | `2026-10-27T17-04-17-350Z-p4640-rcf9e120c55c8be12` | +| `8bfd952f` | `--file test/node/worker-build-lifetime.e2e.test.ts` | passed | `2026-10-27T17-04-25-470Z-p4733-rbbb88b3e4252cfc0` | +| `8bfd952f` | `--file test/node/product-server-proxy.e2e.test.ts` | passed | `2026-10-27T17-04-54-699Z-p4840-r7088235135e9aca5` | +| `d43623ba` | `--file packages/web/test/e2e/theme-product-acceptance.test.ts --grep 'theme settings persist\|executable themes run only'` | failed (2 failing cases) | `2026-10-27T17-06-17-318Z-p5370-rdb44323ae2540294` | +| `d43623ba` | `--file packages/web/test/e2e/browser-demo-mode.test.ts` | failed (3 failing cases) | `2026-10-27T17-06-45-779Z-p5483-r1ecb1a123a8af3bd` | +| `a2fd5a1e` | `--file packages/web/test/e2e/theme-product-acceptance.test.ts --grep 'theme settings persist\|executable themes run only'` | failed (1 failing case) | `2026-10-27T17-08-32-289Z-p6045-r1adf8b65c466ea55` | +| `3156e303` | `--file packages/web/test/e2e/theme-product-acceptance.test.ts --grep 'theme settings persist'` | passed | `2026-10-27T17-10-55-998Z-p6619-r99415de847ed17ad` | +| `3156e303` | `--file packages/web/test/e2e/browser-demo-mode.test.ts` | passed | `2026-10-27T17-11-19-136Z-p6726-rf86e68498d027234` | +| `3156e303` | `--file packages/web/test/e2e/theme-product-acceptance.test.ts` | passed | `2026-10-27T17-11-40-111Z-p6829-r29aa92fa0c6cb29c` | +| `3156e303` | `--file test/repo/build-config-integrity.test.ts --grep 'canonical font preload build seam\|web bundle version stamp'` | passed | `2026-10-27T17-12-32-079Z-p7345-rfbd491c0ee9aed82` | +| `3156e303` | `--file packages/web/test/foundation-copy.test.ts` | passed | `2026-10-27T17-12-48-815Z-p7444-r94b6e463c178a6b4` | +| `3156e303` | `--file test/node/acp-home-context.test.ts` | passed | `2026-10-27T17-12-58-704Z-p7540-r84fc72db15ad1385` | +| `3156e303` | `--file test/node/update-reload.e2e.test.ts` | passed | `2026-10-27T17-13-13-727Z-p7640-rd72830577adb0018` | +| `3156e303` | `--file test/node/update-reload-guard.e2e.test.ts` | passed | `2026-10-27T17-13-31-883Z-p7741-r161ae1bc0a41232f` | +| `3156e303` | `--file packages/web/test/e2e/update-reload.spec.ts` | passed | `2026-10-27T17-14-11-154Z-p7844-r2c4c37ea91b77a0a` | +| `3156e303` | `--file packages/web/test/e2e/update-journey.spec.ts` | passed | `2026-10-27T17-14-29-358Z-p7946-r2c2aef1f919cbe7d` | +| `3156e303` | `--file packages/web/test/e2e/update-notification.01.spec.ts` | passed | `2026-10-27T17-15-05-017Z-p8039-rbee79ef58ccb9edd` | +| `3156e303` | `--file packages/web/test/e2e/update-notification.02.spec.ts` | passed | `2026-10-27T17-15-28-580Z-p8147-r1c56a1d06122b465` | +| `3156e303` | `--file packages/web/test/e2e/gate-boot-barrier.spec.ts` | passed | `2026-10-27T17-15-49-683Z-p8247-r828004637951bbdb` | +| `38976455` | `--file test/repo/worker-builds.test.ts` | passed | `2026-10-27T17-19-37-128Z-p9328-r8f26a6a3c4857187` | +| `38976455` | `--file test/node/product-server-proxy.e2e.test.ts` | incomplete | `2026-10-27T17-19-45-200Z-p9422-r9fa89f239159c80d` | +| `38976455` | `--file test/node/product-server-proxy.e2e.test.ts` | passed | `2026-10-27T17-25-51-595Z-p10092-r14dcdf8329a685aa` | + +Every completed green run above has zero recovered native retries, process leaks and infrastructure failures. One final proxy rerun stalled in the coordinator’s HTTPS `git fetch origin --prune`, before acquisition. After confirming the Git child’s PID, start time, parent and command, I stopped that child on the private coordinator. The canonical result is retained as **incomplete**, with unknown tests and one infrastructure failure; it is neither red nor green evidence. Repeating the identical file selection passed. The repeat used process-local Git low-speed limits (one byte/second for 60 seconds) to bound another network stall; no repository, runner timeout or retry default changed. + +## Request inventories and limits + +The red theme runs observed 254–255 distinct development-path records per application document. On the passing complete walks, the persistence page has 38 distinct request records, and the two executable pages have 27 and 26. The three demo walks have 25, 20 and 20. All six application documents request `/` and built `/assets/` JavaScript; none of their complete retained inventories contains a development path (`/packages/`, `/@vite/`, `/@id/`, `/@fs/` or `/node_modules/`). Inventories omit query values, fragments and headers, and deduplicate by path, resource type and main-frame membership. These are loading-path observations, not raw request counts, peak-memory measurements or incremental savings after PR #32. + +The full theme file on the private Chromium image passed its five Chromium cases and skipped its Firefox-only case. The optional Firefox project was unavailable on that image. This slice makes no new Firefox execution claim. The focused build-config selection covers its four builder/font-preload cases; unrelated CLI telemetry-build cases await the integrated gate. + +Full verification, the killed-worker cleanup backstop, renewable reservations through retrieval, abandoned-run recovery, scratch cleanup and bounded failure diagnostics remain pending under the later slices. The private coordinator and two workers remain allocated for that work. No pool drain, worker recreation, shared-worker experiment or production resource-behavior change was performed. + +Primitive's commit hook surfaced a truncated set of existing decisions; that was not a complete constraint check. No decision was overridden or reconciled. This slice implements the converged direction and establishes no new durable team policy. diff --git a/proofs/arch/artifact-frame/artifact-bridge.md b/proofs/arch/artifact-frame/artifact-bridge.md index 1d9ad2d5..ec5e3e85 100644 --- a/proofs/arch/artifact-frame/artifact-bridge.md +++ b/proofs/arch/artifact-frame/artifact-bridge.md @@ -79,7 +79,7 @@ Readiness assertions observe lifecycle state, never shielding, pointer suspensio - **Contract** (browser-bridge contract harness; the parent is mocked; authored platform identities cover macOS and Windows/Linux): the bridge captures option+arrow on macOS and ctrl+arrow on Windows/Linux for all four arrows, including in text-editing targets and with `repeat`; calls `preventDefault`; and posts `navigation-key`. Shift, every other extra modifier, composition, and non-arrow keys do not forward, while a descendant handler's `stopPropagation()` cannot starve the window capture listener — *(covered by tests in `packages/artifact/test/unit/artifact-bridge.test.ts`: "matches the platform navigation chord for every arrow", "rejects composition, non-arrows, missing chords, and every extra modifier", "captures matching chords in editing targets without stopping propagation", "leaves unmatched keydown behavior untouched", and "produces runnable source that behaves like installBridge when eval'd in a fresh window")*. ^ab-ac-navigation-key - **Contract** (browser-bridge and preload contract harnesses; parent `postMessage` and Electron IPC peers are mocked, forfeiting real handoff to the acceptance assertions below): each producer captures trusted pointer move, down, up, cancel, and click events and emits one `artifact-pointer` carrying the raw guest-viewport coordinates and button fields without preventing default, stopping propagation, or taking pointer capture; untrusted events are ignored — *(covered by `packages/artifact/test/unit/artifact-bridge.test.ts` “reports trusted artifact pointer input without interfering” and `packages/desktop/test/webview-bridge-preload.test.ts` “reports trusted webview pointer input without interfering”)*. ^ab-ac-pointer-producers - **Contract** (host contract harness plus production theme frames in a browser DOM; source windows, frame geometry, and pointer notifications are authored inputs, while browsing-context delivery is forfeited to the acceptance assertions): the current iframe or subscribed webview's source-valid `artifact-pointer` is accepted without readiness, mapped from guest coordinates through the frame's rendered rectangle and layout dimensions, normalized by the theme owner, and delivered once to each current theme frame. A zero layout dimension drops it, and the same payload from another window or replaced webview is ignored — *(covered by `packages/web/test/artifact-view.test.ts` “relays only current artifact pointer input to theme frames in application coordinates”)*. ^ab-ac-pointer-host -- **Acceptance** (outer boundary: physical pointer input in a real Chromium artifact iframe within the built application, served through the real stable-front proxy as harness transport, against a running production server and real theme package; the artifact fixture records its trusted event coordinates without altering input or delivery, and no mocks or production test hooks are used): move, down, up, and click cross the injected bridge and trusted channel to both theme frames in application-viewport coordinates, mapped from the coordinates actually received by the artifact (including Chromium's integer click coordinates), while the artifact's own click action remains effective — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. ^ab-ac-pointer-browser +- **Acceptance** (outer boundary: physical pointer input in a real Chromium artifact iframe within the built application, served through the real stable-front proxy as harness transport, against a running production server and real theme package; the artifact fixture records its trusted event coordinates without altering input or delivery, and no mocks or production test hooks are used): move, down, up, and click cross the injected bridge and trusted channel to both theme frames in application-viewport coordinates, mapped from the coordinates actually received by the artifact (including Chromium's integer click coordinates), while the artifact's own click action remains effective — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^ab-ac-pointer-browser - **Acceptance** (outer boundary: native mouse input in an ordinary HTML artifact webview in the real Electron app against a running production server and real theme package; no mocks or test hooks): the preload and subscribed IPC path deliver move, down, up, and click to both theme frames in application-viewport coordinates while the artifact's own click action remains effective — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “executes an executable theme in Electron and resets on remote consent and selection changes”)*. ^ab-ac-pointer-electron - **Seam** (browser bridge → host navigation handler, crossed by real `postMessage` in the browser seam harness; an authored document and tab/channel state are fixtures, with no mock on the handoff): each of the four forwarded key values reaches the app's navigation handling while focus is inside the document — *(covered by test: `packages/web/test/e2e/artifact-view.test.ts` "forwards every browser navigation arrow from a focused artifact document")*; product outcomes compose with [product/keyboard-navigation.md](../../../specs/product/keyboard-navigation.md). ^ab-ac-navigation-key-seam - **Contract** (desktop shell side of the native-key delivery seam; authored native key events and platform identities cover macOS and Windows/Linux; the renderer navigation receiver is mocked, forfeiting the real handoff to [#^ab-ac-electron-key-seam](#^ab-ac-electron-key-seam)): the matcher implements the same modifier, arrow, repeat, and composition matrix as [#^ab-ac-navigation-key](#^ab-ac-navigation-key), consumes a match, and delivers its arrow value once to app navigation — *(covered by tests: all cases under `packages/desktop/test/native-navigation-key.test.ts` "Electron native navigation chord", plus `packages/desktop/test/main.test.ts` "consumes a matching webview chord and sends its arrow once to the embedding renderer" and `packages/web/test/renderer.test.ts` "delivers Electron native arrows to the existing application navigation edge")*. ^ab-ac-electron-key-contract @@ -116,6 +116,6 @@ The recorder's two branches hand their reports to the host through one function, Demo-mode frames are not bridge participants ([^ab-demo-frame-inert](../../../specs/arch/artifact-frame/artifact-bridge.md#^ab-demo-frame-inert)): -- **Seam** (the built application in Chromium against a really-running server under the server-home hook and cross-origin fixture server declared in [product/artifacts.md](../../product/artifacts.md#test-hooks); two external page artifacts on one channel and a second, newer channel are fixtures created through the production API; code evaluated inside the fixture page's frame posts bridge-shaped `bridge-ready`, a horizontal and a vertical `navigation-key` each chosen to have somewhere to move, `navigation-request`, and `artifact-pointer` messages to its parent through real `postMessage`; the default motion override is the only mock): after the page posts them, the focused channel and selected tab are unchanged, the fixture's first page is still framed at its own URL, and Back and Forward remain absent. Theme pointer delivery is not observed here; [#^ab-ac-demo-inert-host](#^ab-ac-demo-inert-host) carries it — *(policy-grade test for the existing walk: `packages/web/test/e2e/browser-demo-mode.test.ts` "bridge-shaped messages from a demo-mode page change nothing (^ab-ac-demo-inert-seam)"; built-application boundary: test to be written)*. ^ab-ac-demo-inert-seam +- **Seam** (the built application in Chromium against a really-running server under the server-home hook and cross-origin fixture server declared in [product/artifacts.md](../../product/artifacts.md#test-hooks); two external page artifacts on one channel and a second, newer channel are fixtures created through the production API; code evaluated inside the fixture page's frame posts bridge-shaped `bridge-ready`, a horizontal and a vertical `navigation-key` each chosen to have somewhere to move, `navigation-request`, and `artifact-pointer` messages to its parent through real `postMessage`; the default motion override is the only mock): after the page posts them, the focused channel and selected tab are unchanged, the fixture's first page is still framed at its own URL, and Back and Forward remain absent. Theme pointer delivery is not observed here; [#^ab-ac-demo-inert-host](#^ab-ac-demo-inert-host) carries it — *(policy-grade test: `packages/web/test/e2e/browser-demo-mode.test.ts` "bridge-shaped messages from a demo-mode page change nothing (^ab-ac-demo-inert-seam)")*. ^ab-ac-demo-inert-seam - **Contract** (host contract harness; an authored external page artifact posed under browser demo mode, with synthetic `MessageEvent`s whose source is the frame's own window; the transport mock forfeits the real crossing to [#^ab-ac-demo-inert-seam](#^ab-ac-demo-inert-seam)): every message type in the [message protocol](../../../specs/arch/artifact-frame/artifact-bridge.md#message-protocol) produces no navigation-handler, pointer-forwarder, navigation-state, reload, or status-reply call and leaves the frame's trusted identity null — *(covered by test: `packages/web/test/artifact-view.test.ts` "browser demo mode artifact frame > acts on no bridge message from a demo-mode frame (^ab-ac-demo-inert-host)")*. ^ab-ac-demo-inert-host diff --git a/proofs/arch/testing-policy.md b/proofs/arch/testing-policy.md index 9043e7bc..5b9cd3b5 100644 --- a/proofs/arch/testing-policy.md +++ b/proofs/arch/testing-policy.md @@ -12,7 +12,7 @@ Reusable helper builds are runtime behavior owned here. A small contract covers The built-application default is honored by the setup of the product tests that need it. [Themes](../product/themes-and-appearance.md) and [demo mode](../product/artifacts.md#^af-ac-demo-mode) retain their product assertions while changing their asset-loading boundary. No source-text test asserts that every acceptance uses a particular helper. -Killed workers cannot promise teardown. The [Blaxel scratch/recovery assertions](test-runner/blaxel-testshards.md#Lease, recovery, and scratch) carry the backstop rather than duplicating a crash experiment here. The new helper assertions below are not yet discharged. +Killed workers cannot promise teardown. The [Blaxel scratch/recovery assertions](test-runner/blaxel-testshards.md#Lease, recovery, and scratch) carry the backstop rather than duplicating a crash experiment here. ## Test hooks @@ -20,6 +20,6 @@ The shared build-lifetime helper accepts a temporary parent and preparation call ## Worker-owned builds -- **Contract — publish successful builds and dispose owned work.** With a real temporary filesystem and recording preparation callbacks, concurrent requests in one worker reuse a pending/successful build for the same key, while distinct version keys remain distinct. A failed preparation removes its partial output and is not published; a later request may prepare it again. Teardown waits for in-flight preparation to settle and removes all owned outputs, including one that failed after creating files. Per-case server disposal cannot invalidate the worker's reusable build. These callbacks mock build/asset preparation and forfeit actual executable output to the worker seam below. This proves [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime). (test to be written) ^testing-t-build-lifetime +- **Contract — publish successful builds and dispose owned work.** With a real temporary filesystem and recording preparation callbacks, concurrent requests in one worker reuse a pending/successful build for the same key, while distinct version keys remain distinct. A failed preparation removes its partial output and is not published; a later request may prepare it again. Teardown waits for in-flight preparation to settle and removes all owned outputs, including one that failed after creating files. Per-case server disposal cannot invalidate the worker's reusable build. These callbacks mock build/asset preparation and forfeit actual executable output to the worker seam below. This proves [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime). (policy-grade tests: `test/repo/worker-builds.test.ts`, all four cases) ^testing-t-build-lifetime -- **Seam — real runner workers own their builds.** The installed Playwright and Vitest runners each execute a small authored fixture through the actual shared helper/adaptor path, using a private temporary root. Two cases assigned to the same worker reuse the source-CLI preparation and a same-version web bundle; a later worker owns a separate lifetime. Source CLI output starts a real product server, and the real web output is served/read, establishing that published results are usable. Within this same fixture, a bundle path naming an absent child makes the real copy fail after allocation of a per-launch runtime/home; those private outputs are removed, and a subsequent valid launch still uses the worker build. The missing path is fixture data. After per-case server cleanup the shared outputs still exist; after normal worker teardown they do not. No runner, builder, filesystem or server mechanism is mocked. This crosses each framework's teardown boundary under [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime), without a browser or an extra product assertion. [The real version-stamp seam](updates/version-advertisement.md#^t-web-stamp) supplies version breadth; this seam need not rebuild a matrix of versions. (test to be written) ^testing-t-build-worker-seam +- **Seam — real runner workers own their builds.** The installed Playwright and Vitest runners each execute a small authored fixture through the actual shared helper/adaptor path, using a private temporary root. Two cases assigned to the same worker reuse the source-CLI preparation and a same-version web bundle; a later worker owns a separate lifetime. Source CLI output starts a real product server, and the real web output is served/read, establishing that published results are usable. Within this same fixture, a bundle path naming an absent child makes the real copy fail after allocation of a per-launch runtime/home; those private outputs are removed, and a subsequent valid launch still uses the worker build. The missing path is fixture data. After per-case server cleanup the shared outputs still exist; after normal worker teardown they do not. No runner, builder, filesystem or server mechanism is mocked. This crosses each framework's teardown boundary under [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime), without a browser or an extra product assertion. [The real version-stamp seam](updates/version-advertisement.md#^t-web-stamp) supplies version breadth; this seam need not rebuild a matrix of versions. (policy-grade tests: `test/node/worker-build-lifetime.e2e.test.ts`, the Playwright and Vitest cases with their authored native fixtures in `test/fixtures/worker-builds/`) ^testing-t-build-worker-seam diff --git a/proofs/arch/themes/delivery.md b/proofs/arch/themes/delivery.md index b275fe91..9a468515 100644 --- a/proofs/arch/themes/delivery.md +++ b/proofs/arch/themes/delivery.md @@ -6,7 +6,7 @@ Proves [specs/arch/themes/delivery.md](../../../specs/arch/themes/delivery.md). ## Coverage model -The persistence and executable-lifecycle browser walks share the built-application setup declared in [the product coverage model](../../product/themes-and-appearance.md#Coverage model). That setup adaptation remains **(test to be written)**; it changes no delivery outcome and orders no duplicate browser walk here. +The persistence and executable-lifecycle browser walks share the built-application setup declared in [the product coverage model](../../product/themes-and-appearance.md#Coverage model). Their retained request inventories establish that loading boundary; these are the same product walks, with no duplicate browser launch here. Coverage declarations are carried inside the migrated assertion blocks below. The active-route acceptance crosses a really-running server and real package folders for ordinary files, escapes, and validators. JavaScript-route breadth covers all three reserved entries at that same public HTTP boundary: main-entry consent matches and mismatches exact IDs, each manifest declaration is independently enabled, disabled, or absent, and invalid syntax remains fixture bytes rather than being executed. A sibling JavaScript file remains an ordinary static asset. One missing-root recovery walk removes a package with every entry enabled and main consent granted, then proves empty CSS and JavaScript entries, unavailable ordinary assets, and exact restored bytes and validators. The package-root `README.md` exclusion retains its own assertion. @@ -60,7 +60,7 @@ The [visible application-theme seam](#^theme-delivery-t-visible-app) is covered - **Acceptance** (outer boundary: the built application in Chromium and Firefox against a real server and scratch data directory; fixture: an installed theme importing a nested stylesheet with relative day/night SVG URLs; no mocks or test hooks): selecting the theme resolves its registered `--app-wallpaper-image` relative to the declaring stylesheet, loads the correct asset through the application's composed `--app-wallpaper`, changes the image with appearance, and clears it when selecting None — *(covered by test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme-relative wallpaper URLs render through the built app token in Chromium and Firefox”)*. ^theme-delivery-t-wallpaper-url -- **Acceptance** (outer boundary: a real Chromium browser loads the built application and a live-canonical HTML artifact against a really-running server; a selected package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and temporary data directory are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): changing only the nested stylesheet crosses the real `theme-changed` transport and application-link replacement, then changes imported computed presentation in both document classes without registry refresh — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”; built-application boundary: test to be written)*. ^theme-delivery-t-active-package-refresh +- **Acceptance** (outer boundary: a real Chromium browser loads the built application and a live-canonical HTML artifact against a really-running server; a selected package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and temporary data directory are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): changing only the nested stylesheet crosses the real `theme-changed` transport and application-link replacement, then changes imported computed presentation in both document classes without registry refresh — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-delivery-t-active-package-refresh - **Contract** (application theme-resource owner against a mocked DOM, forfeiting browser loading and execution to [the product browser acceptance](../../product/themes-and-appearance.md#^theme-ac-executable-browser)): a refresh creates one cache-busted classic external script in the document head at the connected server's absolute `/theme/main.js`; replacement detaches the prior element before appending the new one, clear removes it, and main-script operations neither replace nor reorder the stylesheet link or sandboxed frames — *(covered by `packages/web/test/theme.test.ts` “installs one cache-busted classic script from the connected server”, “removes the prior script before appending its replacement and clears it”, and “leaves stylesheet and sandboxed frame identity and ordering unchanged”)*. ^theme-delivery-t-app-script-module - **Contract** (renderer composition in a mocked DOM, with connection and application services replaced by controllable boundary mocks, forfeiting real connection event generation and browser resource loading to the browser and Electron assertions below): connection installs the application theme link before display state is ready; a selection change or same-ID resource-refreshing `theme-changed` event replaces it; a consent-membership-only event preserves it; and disconnect removes it — *(covered by test: `packages/web/test/renderer.test.ts` “ties the theme link to connection and theme events before display readiness” and “ties the theme script to confirmed consent, refresh, and connection lifecycle”)*. ^theme-delivery-t-app-link-renderer - **Contract** (application composition against controllable connection and application-service boundaries in a mocked DOM, forfeiting real event transport and script execution to [the product browser acceptance](../../product/themes-and-appearance.md#^theme-ac-executable-browser)): connection leaves the main script absent before confirmed display state; confirmed matching active-ID consent appends the first include only after first-party runtime setup; reconnect reevaluates confirmed state; opt-in injects without replacing the active stylesheet link or frame elements; active-package and registry events with unchanged selection and consent replace the stylesheet and main include; unrelated consent membership preserves all active resource identities; effective appearance changes preserve the stylesheet link and main include while frame replacement follows [the frame-appearance lifecycle](#^theme-delivery-t-frame-appearance-lifecycle); and disconnect removes them. Each path calls the one resource owner rather than using Settings registry state as an execution gate — *(covered by `packages/web/test/renderer.test.ts` “ties the theme script to confirmed consent, refresh, and connection lifecycle” and “reconciles theme frames across confirmed setup, refresh, selection, and connection state”)*. ^theme-delivery-t-app-script-renderer @@ -73,8 +73,8 @@ The [visible application-theme seam](#^theme-delivery-t-visible-app) is covered - **Contract** (application composition against controllable connection, display-state, and registry boundaries in a mocked DOM; deferred registry results and failures are authored outcomes; the mocks forfeit real transport and iframe replacement to [#^theme-delivery-t-frame-browser](#^theme-delivery-t-frame-browser)): selection change and same-ID active-package or registry refresh destroy both prior frames before resolving the current registry, then recreate only enabled destination frames. If initial or replacement lookup fails, both frames remain absent rather than retaining stale executable content; that failure does not interrupt the stylesheet, main-script, or application paths, and reconnect or a later selection, active-package, or registry trigger retries. A resolution or rejection from a superseded connection, theme ID, or lookup can neither install a frame nor remove frames installed by the current lookup. Consent-only events preserve both identities. Null selection and disconnect remove both frames; a registry refresh that disables either declaration leaves that frame absent while independently recreating the other enabled frame; and frame creation, replacement, or removal never requests a full application reload — *(covered by `packages/web/test/renderer.test.ts` “installs ready theme frames after deferred application mount”, “reconciles theme frames across confirmed setup, refresh, selection, and connection state”, “fails theme frame lookup closed while stylesheet, main script, and application continue”, “discards superseded theme frame resolutions and rejections”, and `packages/web/test/theme.test.ts` “remembers ordinary application focus and restores accidental current-frame focus”)*. ^theme-delivery-t-frame-lifecycle - **Contract** (the production application appearance-to-frame composition path in a mocked DOM; an effective root `data-theme` mutation, enabled registry snapshots, deferred fresh lookups, lookup failure, and rapid superseding mutations are authored outcomes; posing the root mutation and replacing the registry and DOM boundaries are mocks that forfeit the real appearance producer and event transport, script execution, and browsing-context replacement to [#^theme-delivery-t-frame-appearance-browser](#^theme-delivery-t-frame-appearance-browser)): under the [sandboxed-frame lifecycle](../../../specs/arch/themes/delivery.md#sandboxed-frames), observing an effective appearance change synchronously removes both current frames in the observer's callback turn before beginning the fresh registry lookup, preserves the active stylesheet and main-script elements without requesting application reload, then recreates every enabled frame for the new value. A failed initial or replacement appearance lookup leaves both absent; a later effective appearance change performs another fresh lookup and can restore enabled frames. A result superseded by a later appearance cannot install stale frames — *(covered by `packages/web/test/renderer.test.ts` “reconciles theme frames across confirmed setup, refresh, selection, and connection state”, “fails theme frame lookup closed while stylesheet, main script, and application continue”, and “discards superseded theme frame resolutions and rejections”)*. ^theme-delivery-t-frame-appearance-lifecycle - **Contract** (production host pointer forwarder against current frame-window and application-event boundary mocks in a browser DOM; the frame windows record `postMessage` and the event methods record interference, forfeiting real dispatch and hit testing to [#^theme-delivery-t-frame-browser](#^theme-delivery-t-frame-browser)): capture-phase listeners forward each trusted application-document `pointermove`, `pointerdown`, `pointerup`, `pointercancel`, and `click` once to each currently installed frame and never to a removed frame. The same internal forwarder accepts a source-authorized artifact input after the artifact owner maps it to application coordinates. Both paths map respectively to `television-theme-pointer-move`, `television-theme-pointer-down`, `television-theme-pointer-up`, `television-theme-pointer-cancel`, and `television-theme-pointer-click`; every message has only `type`, `clientX`, `clientY`, `button`, and `buttons`. Move sends `button: -1` and the input's current `buttons`; down sends the changed input `button` and post-transition `buttons` including the new press; up sends the changed input `button` and post-transition `buttons` excluding the release; click sends the clicked input `button` and `buttons: 0`; and cancel sends `button: -1` and `buttons: 0`, including for an authored cancel input reporting nonzero `buttons`. Each message uses the target frame's current `contentWindow` with destination `"*"`. Untrusted application events are ignored, messages are neither queued nor replayed, and forwarding never prevents default, stops propagation, or takes pointer capture. Any `message` event sourced from either current frame produces no application operation or reply regardless of payload, including an artifact-pointer lookalike — *(covered by `packages/web/test/theme.test.ts` “forwards the closed trusted pointer protocol without interfering or replaying” and “accepts no frame-originated command or reply path”, plus `packages/web/test/artifact-view.test.ts` “relays only current artifact pointer input to theme frames in application coordinates”)*. ^theme-delivery-t-frame-pointer -- **Acceptance** (outer boundary: the built application in Chromium against a really-running server and real package folders; current registry state, production host elements, public entry routes, the injected artifact bridge, and both theme iframe windows remain on the path; the real stable-front proxy supplies harness transport; no mocks or test hooks): independently and jointly enabled entries execute automatically without main consent in separate `allow-scripts` opaque-origin documents, expose their cache-busted route through `document.currentScript.src`, cannot inspect application DOM, and contain load, syntax, and runtime failures. Real pointer move, down, up, cancel, and click input in the application document reaches both frames with the exact viewport coordinates and button fields while the same application sequence and click remain effective; a drag that produces no click forwards only its down, motion, and up transitions. Real move, down, up, and click input in a bridge-capable artifact also reaches both frames in application-viewport coordinates while the artifact's click remains effective. Ordinary application focus changes remain untouched. When either frame unexpectedly acquires focus, focus returns without scrolling to the last connected application element or, when unavailable, to `#app`; the production Settings popover remains operable in its document layer and an application modal dialog remains operable in the browser top layer — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. This is the same browser walk as [product acceptance](../../product/themes-and-appearance.md#^theme-ac-sandboxed-frames), not a second launch. ^theme-delivery-t-frame-browser -- **Acceptance** (outer boundary: the production Settings appearance control in the built Chromium application against a really-running server; a real package enabling both sandbox entries, entry-script startup probes, application-color probes visible through transparent regions of both frames, and a temporary data directory are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): under the [sandboxed-frame appearance contract](../../../specs/arch/themes/delivery.md#sandboxed-frames), an effective light-to-dark appearance change records both old frame removals before either replacement, leaves no old-appearance frame, and reruns both entries. Each replacement document root carries `data-theme="dark"`; its used root scheme and its host element's used scheme are both dark. Rendered-pixel probes behind otherwise empty portions of the background and overlay frames remain visible, proving transparent iframe canvas composition rather than only transparent computed backgrounds — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. This is the same browser walk as [product frame-appearance acceptance](../../product/themes-and-appearance.md#^theme-ac-frame-appearance), not a second launch. ^theme-delivery-t-frame-appearance-browser +- **Acceptance** (outer boundary: the built application in Chromium against a really-running server and real package folders; current registry state, production host elements, public entry routes, the injected artifact bridge, and both theme iframe windows remain on the path; the real stable-front proxy supplies harness transport; no mocks or test hooks): independently and jointly enabled entries execute automatically without main consent in separate `allow-scripts` opaque-origin documents, expose their cache-busted route through `document.currentScript.src`, cannot inspect application DOM, and contain load, syntax, and runtime failures. Real pointer move, down, up, cancel, and click input in the application document reaches both frames with the exact viewport coordinates and button fields while the same application sequence and click remain effective; a drag that produces no click forwards only its down, motion, and up transitions. Real move, down, up, and click input in a bridge-capable artifact also reaches both frames in application-viewport coordinates while the artifact's click remains effective. Ordinary application focus changes remain untouched. When either frame unexpectedly acquires focus, focus returns without scrolling to the last connected application element or, when unavailable, to `#app`; the production Settings popover remains operable in its document layer and an application modal dialog remains operable in the browser top layer — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. This is the same browser walk as [product acceptance](../../product/themes-and-appearance.md#^theme-ac-sandboxed-frames), not a second launch. ^theme-delivery-t-frame-browser +- **Acceptance** (outer boundary: the production Settings appearance control in the built Chromium application against a really-running server; a real package enabling both sandbox entries, entry-script startup probes, application-color probes visible through transparent regions of both frames, and a temporary data directory are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): under the [sandboxed-frame appearance contract](../../../specs/arch/themes/delivery.md#sandboxed-frames), an effective light-to-dark appearance change records both old frame removals before either replacement, leaves no old-appearance frame, and reruns both entries. Each replacement document root carries `data-theme="dark"`; its used root scheme and its host element's used scheme are both dark. Rendered-pixel probes behind otherwise empty portions of the background and overlay frames remain visible, proving transparent iframe canvas composition rather than only transparent computed backgrounds — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. This is the same browser walk as [product frame-appearance acceptance](../../product/themes-and-appearance.md#^theme-ac-frame-appearance), not a second launch. ^theme-delivery-t-frame-appearance-browser - **Acceptance** (outer boundary: the real Electron app connected to a really-running server; an installed theme, its exact computed application-shell color, and an isolated Electron profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): selecting the installed theme renders its application style, and quitting and relaunching with the saved connection renders the same persisted theme without another selection — *(covered by test: `packages/desktop/test/e2e/webview-event-parity.test.ts` “renders the persisted app theme after quitting and relaunching”)*. ^theme-delivery-t-app-link-electron-startup - **Seam** (the production application theme-link module in Chromium against a really-running server on the app's origin and authored theme package resources; no mocks or test hooks): setup, selection, and watched-file refresh keep one cache-busted link last; each replaced link is detached; nested imports, font, and image resources load from the stable path; and appearance changes alter neither link identity nor request count — *(covered by test: `packages/web/test/e2e/appearance-delivery.test.ts` “theme link stays final and mode changes reuse its nested resources”)*. ^theme-delivery-t-app-link - **Acceptance** (outer boundary: the built application and a canonical-v2-styled HTML artifact in Chromium against a really-running server and its installed Clouds package; no mocks or test hooks): the marked app document receives the Clouds wallpaper through the stable route, the artifact keeps its canonical background without requesting or painting that wallpaper, and selecting `None` restores both bare baselines through the null theme while `/theme/theme.css` is empty — *(covered by `packages/web/test/e2e/appearance-delivery.test.ts` “Clouds targets the marked app without painting canonical artifacts”)*. ^theme-delivery-t-app-document diff --git a/proofs/product/artifacts.md b/proofs/product/artifacts.md index 8ddad115..17124fc6 100644 --- a/proofs/product/artifacts.md +++ b/proofs/product/artifacts.md @@ -6,7 +6,7 @@ Proves [specs/product/artifacts.md](../../specs/product/artifacts.md). ## Coverage model -Coverage declarations are carried inside the migrated assertion blocks below. The two demo-mode acceptances and the bridge non-participation seam share the built application and built unsupported-URL view from the real product-server helper. Their setup adaptation is **(test to be written)**; the existing citations name the walks to adapt without adding product assertions or a second browser pass. +Coverage declarations are carried inside the migrated assertion blocks below. The two demo-mode acceptances and the bridge non-participation seam share the built application and built unsupported-URL view from the real product-server helper. Each walk retains a request-event inventory of its built document and assets; these observations use the same browser session as the product assertions. ## Test hooks @@ -51,8 +51,8 @@ Coverage relationship: fresh loading after a channel switch is crossed by [arch/ Browser demo mode ([#^af-demo-mode](../../specs/product/artifacts.md#^af-demo-mode)) is proven by two Chromium acceptance walks, with a routing contract for the breadth of which artifacts it reaches. Shared declaration for the two walks: the outer boundary is Chromium driving the built application and built unsupported-URL view against a really-running server launched with the server-home hook above; the external page is a two-page HTML fixture served by a local HTTP server on an origin other than the Television server's, never a public site; the external page artifact is created through the production API; the default motion override is the only mock. Playwright reads the cross-origin frame's document to observe which fixture page is shown. -- **Acceptance** (shared demo-mode walk): with no marker file, the external page artifact's frame shows the unsupported-URL error page. After the marker file is created and the app is reloaded, the frame's document is the fixture's first page at its own URL, the frame shows no Back or Forward control, and the artifact icon is a button named by [content.yml#return_aria_label](../../specs/ui/app/artifact-frame/content.yml#return_aria_label). After the marker file is deleted and the app is reloaded again, the frame shows the unsupported-URL error page and the icon is not a button — *(policy-grade test for the existing walk: `packages/web/test/e2e/browser-demo-mode.test.ts` "the home-directory marker turns direct external pages on and off at app load (^af-ac-demo-mode)"; built-application boundary: test to be written)*. ^af-ac-demo-mode -- **Acceptance** (shared demo-mode walk, marker present): following the fixture's link shows its second page; pressing the artifact icon shows the first page again. Following the link again and reloading the app shows the first page. Following the link once more, switching to another channel, and returning shows the first page. Back and Forward remain absent throughout — *(policy-grade test for the existing walk: `packages/web/test/e2e/browser-demo-mode.test.ts` "the artifact icon, an app reload, and a channel round trip each return to the original page (^af-ac-demo-return)"; built-application boundary: test to be written)*. ^af-ac-demo-return +- **Acceptance** (shared demo-mode walk): with no marker file, the external page artifact's frame shows the unsupported-URL error page. After the marker file is created and the app is reloaded, the frame's document is the fixture's first page at its own URL, the frame shows no Back or Forward control, and the artifact icon is a button named by [content.yml#return_aria_label](../../specs/ui/app/artifact-frame/content.yml#return_aria_label). After the marker file is deleted and the app is reloaded again, the frame shows the unsupported-URL error page and the icon is not a button — *(policy-grade test: `packages/web/test/e2e/browser-demo-mode.test.ts` "the home-directory marker turns direct external pages on and off at app load (^af-ac-demo-mode)")*. ^af-ac-demo-mode +- **Acceptance** (shared demo-mode walk, marker present): following the fixture's link shows its second page; pressing the artifact icon shows the first page again. Following the link again and reloading the app shows the first page. Following the link once more, switching to another channel, and returning shows the first page. Back and Forward remain absent throughout — *(policy-grade test: `packages/web/test/e2e/browser-demo-mode.test.ts` "the artifact icon, an app reload, and a channel round trip each return to the original page (^af-ac-demo-return)")*. ^af-ac-demo-return - **Contract** (the production artifact-frame view under jsdom with an authored external page artifact and a fake application service posing demo mode on — mocks that forfeit real document loading to [#^af-ac-demo-return](#^af-ac-demo-return); theme-change and artifact-content-change events are injected on that service): neither event reassigns the demo page's frame `src`, which stays the artifact's own URL — *(covered by test: `packages/web/test/artifact-view.test.ts` "browser demo mode artifact frame > leaves a demo-mode page's URL untouched through theme and content change events (^af-ac-demo-untouched)")*. ^af-ac-demo-untouched - **Contract** (the production routing functions `artifactRenderRoute` and `artifactDocumentRoute` over authored artifact records and recorded current URLs; no mocks): with demo mode on in a browser, `http:` and `https:` artifact URLs that are not Television artifact addresses route to a direct iframe at the artifact's own URL, including a URL on the Television host itself; a Television artifact address keeps the shared-artifact route both on another host and on the connected Television host; a non-web URL keeps the unsupported-URL page. Path and Markdown artifacts, a Television-served artifact whose recorded current URL is an external web page, and every artifact in the desktop app route identically with demo mode on and off — *(covered by tests: `packages/web/src/artifact-dispatcher.test.ts` "browser demo mode routing", all four cases)*. ^af-ac-demo-scope diff --git a/proofs/product/themes-and-appearance.md b/proofs/product/themes-and-appearance.md index 91f09e86..605d4336 100644 --- a/proofs/product/themes-and-appearance.md +++ b/proofs/product/themes-and-appearance.md @@ -6,7 +6,7 @@ Proves [specs/product/themes-and-appearance.md](../../specs/product/themes-and-a ## Coverage model -The persistence and executable-lifecycle walks in `theme-product-acceptance.test.ts` load the built application through the real stable-front proxy, keeping the browser origin and authentication across backends that bind port `0` on restart. Their built-application setup adaptation is **(test to be written)**; the citations below identify the existing walks to adapt, not completed evidence of that new boundary. Their product outcomes, fixtures and motion declarations stay the same. The separate `settings.test.ts` registry/control walk keeps its existing development-module boundary. Reset already uses the built application and is unchanged. +The persistence and executable-lifecycle walks in `theme-product-acceptance.test.ts` load the built application through the real stable-front proxy, keeping the browser origin and authentication across backends that bind port `0` on restart. Request-event inventories retained by both walks establish the built document and asset paths across their navigations. Their product outcomes, fixtures and motion declarations stay the same. The separate `settings.test.ts` registry/control walk keeps its existing development-module boundary. Reset already uses the built application and is unchanged. Coverage declarations are carried inside the migrated assertion blocks below. @@ -41,16 +41,16 @@ Clouds' association with Television 1.3 composes with [product versioning's rele ### Acceptance criteria -- **Acceptance** (outer boundaries: real Settings controls over HTTP and websocket connections; the persistence and cross-browser `None`/Clouds walks load the built application, while the separate registry/control walk in `settings.test.ts` retains its development-module boundary; valid, invalid, and dot-prefixed theme folders and real temporary homes are fixtures; the persistence walk uses the real stable-front proxy as harness transport; no mocks or test hooks): a fresh installation offers None plus the themes in [the bundled inventory](../../specs/ui/themes/bundled.yml); refresh shows valid packages and readable invalid-folder errors while every dot-prefixed folder is absent from options and errors; selection propagates to a second client; `None` selects the null theme and clears the active theme ID; the selected theme renders in Chromium and Firefox; and the server-confirmed selection survives a server restart and reconnection in the same browser page — *(policy-grade tests for the existing walks: `packages/web/test/e2e/settings.test.ts` “settings controls cross the registry and confirmed display state” and `packages/web/test/e2e/theme-product-acceptance.test.ts` “the real dropdown renders its selected built-shell theme in Chromium and Firefox” and “theme settings persist rendered app and artifact outcomes through a built-server restart”; the persistence walk’s built-application boundary: test to be written)*. ^theme-ac-selection +- **Acceptance** (outer boundaries: real Settings controls over HTTP and websocket connections; the persistence and cross-browser `None`/Clouds walks load the built application, while the separate registry/control walk in `settings.test.ts` retains its development-module boundary; valid, invalid, and dot-prefixed theme folders and real temporary homes are fixtures; the persistence walk uses the real stable-front proxy as harness transport; no mocks or test hooks): a fresh installation offers None plus the themes in [the bundled inventory](../../specs/ui/themes/bundled.yml); refresh shows valid packages and readable invalid-folder errors while every dot-prefixed folder is absent from options and errors; selection propagates to a second client; `None` selects the null theme and clears the active theme ID; the selected theme renders in Chromium and Firefox; and the server-confirmed selection survives a server restart and reconnection in the same browser page — *(policy-grade tests: `packages/web/test/e2e/settings.test.ts` “settings controls cross the registry and confirmed display state” and `packages/web/test/e2e/theme-product-acceptance.test.ts` “the real dropdown renders its selected built-shell theme in Chromium and Firefox” and “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-ac-selection - **Acceptance** (outer boundary: Firefox drives the real Settings controls in the built application against a really-running production server and its real event transport; fixed-dark and light theme packages, a ready Markdown editor with authored contents, and a temporary home are fixtures; no mocks or test hooks): selecting the light theme changes both the shell and the same loaded Markdown editor from the fixed-dark presentation to the light presentation without replacing the editor document or losing its contents — *(covered by test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “Firefox refreshes a ready Markdown editor from a fixed-dark theme to light”)*. The same browser walk supplies the delivery evidence at [theme-delivery-t-markdown-refresh-firefox](../arch/themes/delivery.md#^theme-delivery-t-markdown-refresh-firefox). ^theme-ac-markdown-refresh-firefox -- **Acceptance** (outer boundary: two real Chromium application documents drive the built application's Settings controls against a running production server and its real event transport; main-enabled, unflagged, syntax-error, and uncaught-exception theme packages, a package-relative asset, an artifact document, and a temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): before consent, the selected `enableMainJS: true` theme has no `main.js` include or request in either application document. Granting exact-ID consent runs its classic main script in both documents after the application runtime is available, exposes `document.currentScript.src`, resolves the package asset from that URL, survives a built-server restart, and leaves the artifact script-free. Appearance changes do not rerun it. An unrelated active-package edit, restoration after complete package-root loss, and server reconnection each rerun it; the missing-root interval executes nothing. An unflagged `main.js` executes nowhere. Separately consented syntax-error and uncaught-exception main scripts report browser errors and leave an ordinary Settings control operational — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. ^theme-ac-executable-browser +- **Acceptance** (outer boundary: two real Chromium application documents drive the built application's Settings controls against a running production server and its real event transport; main-enabled, unflagged, syntax-error, and uncaught-exception theme packages, a package-relative asset, an artifact document, and a temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): before consent, the selected `enableMainJS: true` theme has no `main.js` include or request in either application document. Granting exact-ID consent runs its classic main script in both documents after the application runtime is available, exposes `document.currentScript.src`, resolves the package asset from that URL, survives a built-server restart, and leaves the artifact script-free. Appearance changes do not rerun it. An unrelated active-package edit, restoration after complete package-root loss, and server reconnection each rerun it; the missing-root interval executes nothing. An unflagged `main.js` executes nowhere. Separately consented syntax-error and uncaught-exception main scripts report browser errors and leave an ordinary Settings control operational — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-executable-browser - **Acceptance** (outer boundary: the same two real Chromium application documents and running production server; a main-enabled theme whose script installs document-lifetime probes, a second theme, and persisted Settings-open state are fixtures; no mocks or test hooks): while the main include is active, withdrawing consent, selecting the second theme, and selecting `None` each reload both connected application documents, clear the prior script's document-lifetime effects, and start under the confirmed destination state. Only a document whose Settings popover was open reopens it; its URL marker is consumed before reopening and causes no later reload. After selecting the second theme, selecting the still-consented main-enabled theme again reinstalls its include and executes it without another consent grant. Changing themes while no main include is installed remains in place — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable theme reset reloads connected Chromium documents under confirmed state”)*. ^theme-ac-executable-reset -- **Acceptance** (outer boundary: the same real Chromium application and running production server; a package independently enabling both sandbox entries, a destination theme, syntax-error and uncaught-exception iframe entries, Settings and application modal-dialog controls, and pointer-observation probes are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): without main-page consent, the background and foreground entries execute in separate opaque-origin `allow-scripts` frames at their specified layers and cannot access the application DOM. Theme selection and an unrelated active-package edit replace both frame documents; package-root loss replaces them with empty entries, restoration executes fresh frames, and registry refresh that disables either declaration removes only that frame. Syntax and uncaught runtime errors stay in their affected frames while the application operates. Trusted pointer move, down, up, cancel, and click events in the application document reach both frames with viewport coordinates and button state while the same underlying application sequence, including click, remains effective; a drag that produces no click forwards its down, movement, and up but no click. Pointer move, down, up, and click inside the bridge-capable artifact reach both frames in application-viewport coordinates while the artifact's own click remains effective. Unexpected focus returns first to the last connected application element and then to `#app` when that element is unavailable. The foreground layers do not prevent using the production Settings popover in its document layer or an application modal dialog in the browser top layer — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. ^theme-ac-sandboxed-frames -- **Acceptance** (outer boundary: the production Settings appearance control in a real Chromium application against a running production server; a real active package enabling both sandbox entries, entry-script startup probes, visible application probes behind transparent regions of both frames, and a temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): under the [frame lifecycle](../../specs/product/themes-and-appearance.md#^theme-iframe-lifecycle), an effective light-to-dark appearance change removes both old frame documents before installing replacements, leaves no old-appearance frame, and reruns both entries in documents whose root `data-theme` is `dark`. Each replacement host and document root has a used dark scheme, and the application probes remain visibly unchanged through the frames' transparent regions — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. ^theme-ac-frame-appearance -- **Acceptance** (outer boundary: a real Chromium browser loads the built application and a live-canonical HTML artifact against a running built server and its real `/events` transport; a real active package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and a temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): changing only the nested stylesheet updates the application and artifact presentation; moving the complete active package away produces their foundation and canonical baselines while the exact selected theme ID remains confirmed; restoring it at the same path restores both presentations and keeps that ID, without registry refresh or another selection — *(policy-grade test for the existing walk: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”; built-application boundary: test to be written)*. ^theme-ac-active-package +- **Acceptance** (outer boundary: the same real Chromium application and running production server; a package independently enabling both sandbox entries, a destination theme, syntax-error and uncaught-exception iframe entries, Settings and application modal-dialog controls, and pointer-observation probes are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): without main-page consent, the background and foreground entries execute in separate opaque-origin `allow-scripts` frames at their specified layers and cannot access the application DOM. Theme selection and an unrelated active-package edit replace both frame documents; package-root loss replaces them with empty entries, restoration executes fresh frames, and registry refresh that disables either declaration removes only that frame. Syntax and uncaught runtime errors stay in their affected frames while the application operates. Trusted pointer move, down, up, cancel, and click events in the application document reach both frames with viewport coordinates and button state while the same underlying application sequence, including click, remains effective; a drag that produces no click forwards its down, movement, and up but no click. Pointer move, down, up, and click inside the bridge-capable artifact reach both frames in application-viewport coordinates while the artifact's own click remains effective. Unexpected focus returns first to the last connected application element and then to `#app` when that element is unavailable. The foreground layers do not prevent using the production Settings popover in its document layer or an application modal dialog in the browser top layer — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-sandboxed-frames +- **Acceptance** (outer boundary: the production Settings appearance control in a real Chromium application against a running production server; a real active package enabling both sandbox entries, entry-script startup probes, visible application probes behind transparent regions of both frames, and a temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): under the [frame lifecycle](../../specs/product/themes-and-appearance.md#^theme-iframe-lifecycle), an effective light-to-dark appearance change removes both old frame documents before installing replacements, leaves no old-appearance frame, and reruns both entries in documents whose root `data-theme` is `dark`. Each replacement host and document root has a used dark scheme, and the application probes remain visibly unchanged through the frames' transparent regions — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-frame-appearance +- **Acceptance** (outer boundary: a real Chromium browser loads the built application and a live-canonical HTML artifact against a running built server and its real `/events` transport; a real active package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and a temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): changing only the nested stylesheet updates the application and artifact presentation; moving the complete active package away produces their foundation and canonical baselines while the exact selected theme ID remains confirmed; restoring it at the same path restores both presentations and keeps that ID, without registry refresh or another selection — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-ac-active-package - **Acceptance** (outer boundary: the real Electron app and another client connected to a running server; a theme enabling main, background, and foreground entries, a second theme, an artifact webview, and an isolated Electron profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): before consent the main script does not execute, while both sandboxed frames are created and execute in the application. Pointer move, down, up, and click inside the ordinary HTML artifact webview reach both theme frames in application-viewport coordinates while the artifact's click remains effective. Granting consent through Settings executes `main.js` in the application renderer but no theme JavaScript in the artifact webview, and the renderer exposes the same Node-facing globals and preload bridge before and after activation. Consent survives application restart; remote main opt-out and remote selection of the second theme each reload the application document, clear its document-lifetime probe, and apply confirmed state. A remote same-theme active-package event replaces both frame documents without itself causing a full application reload — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “executes an executable theme in Electron and resets on remote consent and selection changes”)*. ^theme-ac-executable-electron - **Acceptance** (outer boundary: the real Electron app connected to a running server; an installed theme and isolated Electron profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): the selected application theme renders again after quitting and relaunching through the saved connection — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “renders the persisted app theme after quitting and relaunching”)*. ^theme-ac-selection-electron -- **Acceptance** (outer boundary: the built application in a real Chromium browser against a really-running server; one adaptive package and one fixed-dark package, local Television-managed HTML and markdown artifacts, a third-party URL artifact, both sandboxed entries with startup probes, dedicated computed-style probes, controlled system appearance, and a real temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): under the adaptive theme, `Adapt to system`, `Light`, and `Dark` produce their expected effective appearance in the application and applicable artifacts. Under the fixed-dark theme, selecting and persisting an opposing Light preference leaves the application root, foundation mode values and native scheme, both artifact classes, and both frame host/document pairs dark; it does not reload app or artifacts, and it keeps the existing frame documents because effective appearance did not change. The same fixed results survive a built-server restart. Returning to the adaptive theme applies the latest Light preference; that effective change replaces and reruns both frames while leaving their transparent composition intact — *(policy-grade tests for the existing walks: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart” and “executable themes run only in the Chromium application lifecycle and isolate failures”; built-application boundary: test to be written)*. ^theme-ac-appearance +- **Acceptance** (outer boundary: the built application in a real Chromium browser against a really-running server; one adaptive package and one fixed-dark package, local Television-managed HTML and markdown artifacts, a third-party URL artifact, both sandboxed entries with startup probes, dedicated computed-style probes, controlled system appearance, and a real temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): under the adaptive theme, `Adapt to system`, `Light`, and `Dark` produce their expected effective appearance in the application and applicable artifacts. Under the fixed-dark theme, selecting and persisting an opposing Light preference leaves the application root, foundation mode values and native scheme, both artifact classes, and both frame host/document pairs dark; it does not reload app or artifacts, and it keeps the existing frame documents because effective appearance did not change. The same fixed results survive a built-server restart. Returning to the adaptive theme applies the latest Light preference; that effective change replaces and reruns both frames while leaving their transparent composition intact — *(policy-grade tests: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart” and “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-appearance - **Acceptance** (outer boundary: the real Electron app connected to a really-running server; one adaptive and one fixed-dark installed package, an existing Television-managed artifact webview, a third-party URL artifact webview, native-appearance observation, computed-style probes, and an isolated profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): the adaptive theme carries `system`, `light`, and `dark` through `nativeTheme` to both retained webviews and their media-query results. Under the fixed-dark theme, an opposing Light preference persists while the application, `nativeTheme`, native-surface appearance source, and both current webviews remain dark and loaded. Returning to the adaptive theme applies that latest Light preference; the third-party URL webview stays loaded, while the Television-managed artifact follows its ordinary theme-selection reload lifecycle — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “applies manifest-resolved appearance to Electron and retained webviews”)*. This is the same Electron walk as [desktop appearance acceptance](../arch/desktop/appearance.md#^desktop-appearance-t-electron), not a second launch. ^theme-ac-appearance-electron - **Acceptance** (outer boundary: a real browser loads artifacts from a really running production `Server`; the committed old-style v1 document, an authored v2 control document, fresh production canonical tree, temporary storage, and installed loud unscoped theme are fixtures; no mocks or test hooks): the v1 artifact loads its wrapper, base, components, and font but makes no theme request, renders the fixed light canonical probe, and does not render the loud theme value; the v2 control requests the theme and renders its loud value and dark scheme — *(covered by `packages/server/test/e2e/canonical.spec.ts` “frozen v1 stays fixed light while live v2 receives the active theme”)*. ^theme-ac-frozen-v1 - **Acceptance** (outer boundary: a spawned built CLI server against a real fresh temporary home; the packaged bundled-theme tree is the fixture; no mocks or test hooks): first boot installs every bundled theme byte-for-byte, selects Clouds from an unset theme, exposes its exact stylesheet and wallpaper assets, and records the one-time decision — *(covered by `test/node/theme-packaging.test.ts` “Built CLI installs every packaged theme and initially selects Clouds”)*. ^theme-ac-bundled From bda9b6cd7936413a13470c358466a882a4cf087a Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:47:35 +0000 Subject: [PATCH 20/64] Specify bounded resource diagnostics and real parent capture --- ...arness-resource-cleanup-review-response.md | 6 ++ ...ness-resource-cleanup-slice-1-review-r1.md | 57 +++++++++++ .../test-harness-resource-cleanup-slice-1.md | 2 +- proofs/arch/testing-policy.md | 2 +- scripts/test/resource-diagnostics.d.mts | 2 + scripts/test/resource-diagnostics.mjs | 17 ++++ .../resource-diagnostics/native.fixture.ts | 8 ++ .../fixtures/resource-diagnostics/prepare.mjs | 8 ++ test/fixtures/resource-diagnostics/setup.mjs | 3 + test/repo/resource-diagnostics-links.test.ts | 56 +++++++++++ test/repo/resource-diagnostics.test.ts | 85 ++++++++++++++++ test/repo/test-runner-resource-parent.test.ts | 96 +++++++++++++++++++ 12 files changed, 340 insertions(+), 2 deletions(-) create mode 100644 docs/working/test-harness-resource-cleanup-slice-1-review-r1.md create mode 100644 scripts/test/resource-diagnostics.d.mts create mode 100644 scripts/test/resource-diagnostics.mjs create mode 100644 test/fixtures/resource-diagnostics/native.fixture.ts create mode 100644 test/fixtures/resource-diagnostics/prepare.mjs create mode 100644 test/fixtures/resource-diagnostics/setup.mjs create mode 100644 test/repo/resource-diagnostics-links.test.ts create mode 100644 test/repo/resource-diagnostics.test.ts create mode 100644 test/repo/test-runner-resource-parent.test.ts diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 2b968b1e..c9efaec3 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -210,3 +210,9 @@ Four contract cases and the real two-framework seam have intended red/green evid All execution occurred on the named private Blaxel hosts at pushed revisions; nothing ran on the shared development host and no shared worker was modified. The five mirrored proof files now cite the implemented, passing coverage. The optional Firefox project was unavailable; no new Firefox validation is claimed. Slice 1's killed-worker backstop, diagnostic transport, renewable worker ownership and recovery remain assigned to slices 2 and 3. Full verification and integrated review are still pending. The private hosts remain allocated for those slices. The supervisor arranges independent review of this slice before further implementation. No new durable team decision was made; this work implements the converged proposal, specs, proofs and plan. + +## Slice 1 round 1: convergence and slice 2 + +The supervisor relayed **PASS** at `036d0491`; the [complete review](test-harness-resource-cleanup-slice-1-review-r1.md) is preserved unchanged. Apply the proof-only refinement by attributing per-case server disposal to the real seam, and clarify that the final static log ends with the separately recorded fetch interruption. Keep the second framework invocation: it directly exercises the reviewed later-worker lifetime assertion, and removing it now would change that proof's scope. This optional cost reduction does not block convergence. + +The reported 143 deployed shared sandboxes remain an operator question for Josh and an integrated-review documentation item. Do not infer quota semantics or change the pool. Slice 2 implements bounded parent diagnostics, retained links and retry history; live provider recovery/diagnostic qualification and the lease/download race remain explicitly pending until slice 3. diff --git a/docs/working/test-harness-resource-cleanup-slice-1-review-r1.md b/docs/working/test-harness-resource-cleanup-slice-1-review-r1.md new file mode 100644 index 00000000..f3daa46e --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-slice-1-review-r1.md @@ -0,0 +1,57 @@ +# Slice 1 implementation review, round 1: test-harness resource cleanup (PR #12, 036d0491) + +**Verdict: PASS.** Slice 1 delivers what the plan assigned it: reusable source-CLI and versioned web builds now belong to the runner worker that created them, are published only after preparation succeeds, survive per-case cleanup and are removed by real Playwright and Vitest worker teardown; failed preparation and a failed per-launch copy leave nothing behind; and the five selected walks load the built application through the product server, with the two theme walks keeping their origin across port-0 restarts through the stable-front proxy. The tests prove what the proofs order, at the declared shapes, with no mock on the paths that matter. The validation record is accurate: every row of its run table matches the run directory it names, by commit, counts and outcome, and its two non-green events (a setup failure and a stalled fetch) are described as exactly what they were. No production file changed. Three refinements follow, none blocking, plus one observation outside the slice that the supervisor should see. + +- Reviewed: `git diff 04a6b275 036d0491` in full (helpers, fixtures, the two new tests, the five converted walks, eleven registration edits, the proof regrades, the record); the plan's slice 1; `^worker-build-lifetime`, `^product-acceptance-built-app` and the testing policy's mocking, shape and cost rules; the proof assertions the tests cite; the server's static and view routing; the runner's registry; and the retained run directories and static logs the record names, which exist in this worktree. +- Branch state: contains `origin/main` (`a8e73fbf`); the copied round 2 plan review is identical to mine; the plan edits since `04a6b275` are the two round 2 refinements and a status update. +- I ran nothing. I read run summaries, native reports and logs with small scripts; nothing was built, tested or changed. + +## What the slice proves, against the proofs + +**`^testing-t-build-lifetime` (contract, `test/repo/worker-builds.test.ts`).** Four cases over a real temporary filesystem with recording builders: a concurrent request for the same key gets the same pending promise and the builder runs once; distinct keys get distinct directories; a failed preparation removes its partial output, is not published and permits a fresh build; teardown waits for pending preparation, fulfilled or rejected, then leaves the root empty. These match the contract's text, with one clause that belongs elsewhere (refinement 1). Red evidence: four failures at `2460d06a` on retained output; green at `8bfd952f`. + +**`^testing-t-build-worker-seam` (seam, `test/node/worker-build-lifetime.e2e.test.ts`).** Each of Playwright and Vitest is spawned for real on an authored two-case fixture under a private `TMPDIR`, with configs generated into that directory so the registry sees no unregistered surface (the first attempt's validation failure at `564a8edb` is correctly recorded as setup, not red). The fixture starts a real product server from the source-CLI build and fetches its document and a script; builds a real versioned web bundle and reads its entry; asserts one shared source build after case cleanup, the bundle still present and the case's home gone; and in its first case makes the per-launch copy fail on an absent bundle and asserts no `television-product-e2e-*` remains before a valid launch. The parent asserts exit 0, two observations identical but for case number, no owned-process survivor, and no `television-source-cli-e2e-*`, `television-web-bundle-*` or `television-product-e2e-*` left in the private root after teardown. That is the crossing the proof orders: real runner, real builders, real server, real framework teardown, nothing mocked. Red at `74485d22` and `2460d06a` (four retained directories), green at `8bfd952f`. + +**The implementation behind them.** `WorkerBuilds` (`test/helpers/worker-builds.ts:18-44`) creates and registers the directory before the builder runs, publishes the directory only when the builder's promise resolves, deletes the key and the directory on rejection, refuses new requests once disposal has begun, and waits for every pending build before removing. `getSourceCLIEntry()` now returns its entry only after the build and the runtime-asset copy succeed (`test/helpers/product-server.ts:69-77`), which is the original leak closed at its cause. The web builder keeps its version keys and serialized environment override. Per-launch cleanup is registered before configuration or preparation can fail; disposal during preparation waits for it rather than racing the home's removal; a failed bundle copy removes its own run directory (`:85-105`, `:221-230`). Playwright's shared registration gains an automatic worker fixture that disposes servers first and removes builds only if that succeeded, so a server that could not be stopped keeps its files (`test/helpers/playwright.ts:8-17`); Vitest callers get the same through `test/helpers/vitest.ts`. Every caller of `launchProductServer` or `buildVersionedWebBundle` in the tree imports one of the two registrations (38 callers; the only exception is the fixture body, which the fixture files cover). + +**The five walks (`^product-acceptance-built-app`; the regraded theme, delivery, artifact and bridge citations).** Demo mode navigates to the server URL and asserts the built boundary on every load; the theme walks navigate to the stable-front proxy, retarget it after `restart()`, and assert the boundary after the first shell. `recordApplicationRequests` records main-frame request paths without query values or headers, asserts no `/packages/`, `/@vite/`, `/@id/`, `/@fs/` or `/node_modules/` path, a root document and a built `/assets/*.js` script, and attaches the inventory on exit, pass or fail. I decoded the attachments from the green full-file theme run: two inventories, 27/26 and 38 records, three root documents, zero development paths, as the record states. The unsupported-URL view is now copied under the runtime's static web root, which is where production serves it: the server mounts only `artifact-missing` and `markdown` explicitly and serves everything else from `staticDir`, into which the CLI build copies the web dist with its `views/url-unsupported` (`packages/server/src/server.ts:320-339`). Red at `d43623ba` on all five cases (development modules requested), the persistence 404 at `a2fd5a1e` fixed by that placement, green at `3156e303`; the reset walk and `settings.test.ts` are untouched. + +**Regrades.** Each changed citation now names the test that passed, and the coverage models drop their "to be written" notes. The full theme file's one skip is the Firefox Markdown case, which the web Playwright config skips when the Firefox executable is absent, and the record says so. + +## The record, checked + +| Claim | Evidence | +| --- | --- | +| 25 runs, each at the named commit with the named result | All 25 summaries present locally; `git.commit` matches each row; counts match (`testsFailed` 1/4/1/2/3/1 on the red rows; zero failures, zero recovered retries, zero leaks, zero infra failures on every green row). | +| The incomplete proxy run was a stalled coordinator fetch, not a test result | Its summary: `surfacesRun 0`, `infraFailures 1`; the static log shows `git-remote-https died of signal 15` after the author stopped the child. The repeat passed. | +| Lint and type-check passed at `38976455`; `502ba255` had one type error | `.test-runs/slice-1-static/502ba255.log` shows the `intervals` error; `38976455.log` shows both commands clean before the two final Blaxel runs. | +| Default retry budgets kept | The coordinator lines show `--target-retries 0` for `unit:root` and `--test-retries 2` for `e2e:node`, with `--retry-infra 2`. | +| Nothing on the shared host; private pool only | Every coordinator line names `--pool rose-cleanup`; every path is under `/workspace/television` on the coordinator. | + +## Refinements (none blocking) + +1. **One contract clause is proven by the seam, not the contract.** `^testing-t-build-lifetime` says "Per-case server disposal cannot invalidate the worker's reusable build", but `test/repo/worker-builds.test.ts` has no server; the fixture body of the seam proves it (shared outputs present after `server.dispose()`). Move the sentence to `^testing-t-build-worker-seam`, or cite the seam for it. Proof-only. +2. **The seam runs each framework twice; once would prove the same.** The second nested run exists to show "a later worker owns a separate lifetime", but its separate paths follow from its private `TMPDIR`, and the lifetime claim that matters, nothing left after teardown, is proven by the first run. Dropping the loop halves four source-CLI builds and four Vite builds per execution of this test. Optional; the plan accepted a real build and the surface is remote-only. +3. **One line in the record for the last static log.** `38976455.log` ends with the stopped fetch's stack trace, which a reader opening it cold could take for a static-check failure. The record already explains the event in prose; a clause pointing at it next to "Static logs are retained" would save the reader the inference. + +## Outside the slice, for the supervisor + +The response records a read-only provider inventory of **143 deployed `poc` sandboxes** against the 72 the Blaxel spec documents (`blaxel-testshards.md:25`). Slice 1 is unaffected, but it bears on two things already in play: the plan's tier arithmetic (idle sandboxes evidently do not count against the 75-concurrent-shard limit, or the limit is not what the spec says) and the spec's pool-size statement, which is stale either way and should be reconciled at the integrated review. Whether 71 extra sandboxes are intended capacity or accumulated residue is a question for Josh or whoever operates the pool, not for this contribution. + +## Reviewer checklist + +- **Proof coverage.** Both slice obligations discharged at their declared shapes with the cited tests; the eleven regraded walk citations name passing tests; coverage models updated. +- **Mocking and boundaries.** The contract's builders are the declared substitutes; the seam and the walks have no mock. Clearing `VITEST` for spawned servers is not a mock: it removes the guard that would stop the CLI's entry under Vitest (`packages/cli/src/index.ts:2283`) and is used by ten existing tests on `main`. +- **Honesty of evidence.** The record distinguishes setup failures, red evidence, stalled infrastructure and skips, and claims no Firefox execution. +- **Scope.** Test helpers, fixtures, tests, proofs and working documents only; no production, dependency, timeout or retry change. +- **Baseline.** Green for its obligations; the killed-worker backstop remains assigned to slice 3 as the plan states. + +## Evidence and limits + +- Read every changed file and the full current `worker-builds.ts`, `product-server.ts` launch path, both registrations, the request helper, the fixtures and both tests; cross-checked all 25 run summaries, two native reports and two static logs; decoded the request-inventory attachments. +- Not established: anything about the private workers beyond what the logs show; the exact Vitest behavior of a worker-scoped fixture across files under `pool: forks` (the seam's green run on both runners is the evidence that teardown fires where it must). + +## Housekeeping + +- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel beyond scripts that read JSON and logs already on disk. `git fetch origin` updated remote-tracking refs. +- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-slice-1.md b/docs/working/test-harness-resource-cleanup-slice-1.md index b6ddc766..4e1067ab 100644 --- a/docs/working/test-harness-resource-cleanup-slice-1.md +++ b/docs/working/test-harness-resource-cleanup-slice-1.md @@ -21,7 +21,7 @@ Every test ran from `/workspace/television` on `tv-cleanup-coordinator-rose`, us - The four cache contracts at `2460d06a` failed on retained partial/completed output. Their builders are the proof-declared substitutes and use a real temporary filesystem. The lifecycle seam, rather than these callbacks, establishes usable real CLI/web output and actual framework teardown. - At `8bfd952f`, all four contracts, both framework seam cases, and the existing startup-disposal/proxy file passed. Each framework ran two worker lifetimes; cases reused output within a worker, later workers had separate paths, failed-copy output disappeared before the valid launch, and teardown left no owned build or launch directories. - At `d43623ba`, all five selected browser cases reached and failed the new request-boundary assertion: development modules were still requested. At `a2fd5a1e`, the executable walk passed but persistence failed on a 404 for the unsupported-URL view. The helper had copied it to the sibling-view location even though the server serves this document through the web static root. `3156e303` corrects only that destination. The narrow persistence run, whole demo file and whole theme file then passed. -- Inspection of the repository TypeScript target prompted `502ba255`: explicit constructor field assignments replace parameter properties, and test rendezvous promises use ES2022-compatible construction. This changes no lifetime behavior. Remote lint passed at that revision; type checking found that the adapted startup-disposal assertion had used Playwright’s plural `intervals` instead of Vitest’s singular `interval`. `38976455` corrects the option. Both `npm run lint` and `npm run type-check` then passed on the private coordinator at that pushed revision, and the cache-contract and startup-disposal files passed there through the canonical Blaxel runner. Static logs are retained locally under `.test-runs/slice-1-static/`. +- Inspection of the repository TypeScript target prompted `502ba255`: explicit constructor field assignments replace parameter properties, and test rendezvous promises use ES2022-compatible construction. This changes no lifetime behavior. Remote lint passed at that revision; type checking found that the adapted startup-disposal assertion had used Playwright’s plural `intervals` instead of Vitest’s singular `interval`. `38976455` corrects the option. Both `npm run lint` and `npm run type-check` then passed on the private coordinator at that pushed revision, and the cache-contract and startup-disposal files passed there through the canonical Blaxel runner. Static logs are retained locally under `.test-runs/slice-1-static/`; `38976455.log` ends with the later stopped-fetch stack trace described below, after lint and type checking had passed. The run table below records the native evidence and exact canonical selections. Each command has the prefix `npm test -- blaxel --pool rose-cleanup`; the `--run-dir-output` receipt used for orchestration changes no selection. Every run directory is under `/workspace/television/.test-runs/` on the coordinator and has also been copied under `.test-runs/` in the contribution worktree. `summary.json`, `results.json`, `events.ndjson`, the provider request/report/setup/shard files and native JSON are retained for executed runs. The interrupted fetch has only canonical summaries/events and its coordinator log; it never produced a provider or native report. The native JSON embeds request inventories, so they remain available after later worker checkout or cleanup. diff --git a/proofs/arch/testing-policy.md b/proofs/arch/testing-policy.md index 5b9cd3b5..78774cef 100644 --- a/proofs/arch/testing-policy.md +++ b/proofs/arch/testing-policy.md @@ -20,6 +20,6 @@ The shared build-lifetime helper accepts a temporary parent and preparation call ## Worker-owned builds -- **Contract — publish successful builds and dispose owned work.** With a real temporary filesystem and recording preparation callbacks, concurrent requests in one worker reuse a pending/successful build for the same key, while distinct version keys remain distinct. A failed preparation removes its partial output and is not published; a later request may prepare it again. Teardown waits for in-flight preparation to settle and removes all owned outputs, including one that failed after creating files. Per-case server disposal cannot invalidate the worker's reusable build. These callbacks mock build/asset preparation and forfeit actual executable output to the worker seam below. This proves [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime). (policy-grade tests: `test/repo/worker-builds.test.ts`, all four cases) ^testing-t-build-lifetime +- **Contract — publish successful builds and dispose owned work.** With a real temporary filesystem and recording preparation callbacks, concurrent requests in one worker reuse a pending/successful build for the same key, while distinct version keys remain distinct. A failed preparation removes its partial output and is not published; a later request may prepare it again. Teardown waits for in-flight preparation to settle and removes all owned outputs, including one that failed after creating files. Per-case server disposal is covered by the real worker seam below. These callbacks mock build/asset preparation and forfeit actual executable output to the worker seam below. This proves [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime). (policy-grade tests: `test/repo/worker-builds.test.ts`, all four cases) ^testing-t-build-lifetime - **Seam — real runner workers own their builds.** The installed Playwright and Vitest runners each execute a small authored fixture through the actual shared helper/adaptor path, using a private temporary root. Two cases assigned to the same worker reuse the source-CLI preparation and a same-version web bundle; a later worker owns a separate lifetime. Source CLI output starts a real product server, and the real web output is served/read, establishing that published results are usable. Within this same fixture, a bundle path naming an absent child makes the real copy fail after allocation of a per-launch runtime/home; those private outputs are removed, and a subsequent valid launch still uses the worker build. The missing path is fixture data. After per-case server cleanup the shared outputs still exist; after normal worker teardown they do not. No runner, builder, filesystem or server mechanism is mocked. This crosses each framework's teardown boundary under [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime), without a browser or an extra product assertion. [The real version-stamp seam](updates/version-advertisement.md#^t-web-stamp) supplies version breadth; this seam need not rebuild a matrix of versions. (policy-grade tests: `test/node/worker-build-lifetime.e2e.test.ts`, the Playwright and Vitest cases with their authored native fixtures in `test/fixtures/worker-builds/`) ^testing-t-build-worker-seam diff --git a/scripts/test/resource-diagnostics.d.mts b/scripts/test/resource-diagnostics.d.mts new file mode 100644 index 00000000..957d79db --- /dev/null +++ b/scripts/test/resource-diagnostics.d.mts @@ -0,0 +1,2 @@ +export function startResourceDiagnostics(options: { root: string; identity: Record; readProbe?: (name: string, interval?: Record) => unknown; now?: () => Date; write?: (file: string, contents: string) => void; warn?: (message: string) => void }): { file: string | null; finish: (outcome: { runner: string; nativeResultPath: string | null; attemptResultPath?: string | null; exitCode: number; incomplete?: boolean }) => string[] }; +export function writeResourceDiagnostic(root: string, name: string, record: Record, options?: { write?: (file: string, contents: string) => void; warn?: (message: string) => void }): string | null; diff --git a/scripts/test/resource-diagnostics.mjs b/scripts/test/resource-diagnostics.mjs new file mode 100644 index 00000000..1f42c079 --- /dev/null +++ b/scripts/test/resource-diagnostics.mjs @@ -0,0 +1,17 @@ +import fs from "node:fs"; +import path from "node:path"; + +// Initial public boundary for the red collector/writer contracts. +export function startResourceDiagnostics({ root, identity }) { + const file = path.join(root, "resources", `${identity.surfaceId.replace(/[^a-zA-Z0-9-]/g, "-")}.json`); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify({ schemaVersion: 1 })); + return { file, finish: () => [file] }; +} + +export function writeResourceDiagnostic(root, name, record) { + const file = path.join(root, "resources", `${name}.json`); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(record)); + return file; +} diff --git a/test/fixtures/resource-diagnostics/native.fixture.ts b/test/fixtures/resource-diagnostics/native.fixture.ts new file mode 100644 index 00000000..9254bdc3 --- /dev/null +++ b/test/fixtures/resource-diagnostics/native.fixture.ts @@ -0,0 +1,8 @@ +import { afterAll, expect, test } from "vitest"; +import { writeFileSync } from "node:fs"; +import path from "node:path"; + +afterAll(() => writeFileSync(path.join(process.env.RESOURCE_FIXTURE_ROOT!, "teardown.json"), JSON.stringify({ observedAt: new Date().toISOString() }))); +test("retains a failed attempt before recovery", (context) => { + expect(context.task.result?.retryCount ?? 0).toBeGreaterThan(0); +}); diff --git a/test/fixtures/resource-diagnostics/prepare.mjs b/test/fixtures/resource-diagnostics/prepare.mjs new file mode 100644 index 00000000..57037663 --- /dev/null +++ b/test/fixtures/resource-diagnostics/prepare.mjs @@ -0,0 +1,8 @@ +import fs from "node:fs"; +import path from "node:path"; + +const root = process.env.RESOURCE_FIXTURE_ROOT; +const resources = path.join(root, "results", "resources"); +const records = fs.existsSync(resources) ? fs.readdirSync(resources).filter((name) => name.endsWith(".json")).map((name) => JSON.parse(fs.readFileSync(path.join(resources, name), "utf8"))) : []; +fs.writeFileSync(path.join(root, "preparation.json"), JSON.stringify({ observedAt: new Date().toISOString(), records })); +if (process.env.RESOURCE_FIXTURE_MODE === "interrupt") await new Promise((resolve) => setTimeout(resolve, 30_000)); diff --git a/test/fixtures/resource-diagnostics/setup.mjs b/test/fixtures/resource-diagnostics/setup.mjs new file mode 100644 index 00000000..e1930cac --- /dev/null +++ b/test/fixtures/resource-diagnostics/setup.mjs @@ -0,0 +1,3 @@ +export default function setup() { + if (process.env.RESOURCE_FIXTURE_MODE === "exit") process.exit(17); +} diff --git a/test/repo/resource-diagnostics-links.test.ts b/test/repo/resource-diagnostics-links.test.ts new file mode 100644 index 00000000..67a90d22 --- /dev/null +++ b/test/repo/resource-diagnostics-links.test.ts @@ -0,0 +1,56 @@ +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { expect, test } from "vitest"; +import { combineAttemptReports, compactProviderSummary, summarizeInfrastructureAttempts } from "../../scripts/test/provider-outputs.mjs"; +import { normalizeProviderShardSurfaces } from "../../scripts/test/provider-normalization.mjs"; +import { buildReports } from "../../scripts/test/reporting.mjs"; +import { buildTimingEvents, serializeTimingEvents } from "../../scripts/test/timing-events.mjs"; + +const surfaces = [{ id: "e2e:fixture", runner: "playwright", roots: ["probe.test.ts"], cwd: "." }] as any; +const start = "2026-10-27T00:00:00.000Z"; +const end = "2026-10-27T00:00:01.000Z"; + +test("keeps only retained resource links across infrastructure replacement, normalization and final reporting", () => { + const root = mkdtempSync(path.join(os.tmpdir(), "tv-resource-links-")); + try { + const attempts = [0, 1].map((index) => { + const dir = path.join(root, `attempt-${index}`); mkdirSync(dir); + const diagnostic = path.join(dir, "resources.json"); + writeFileSync(diagnostic, JSON.stringify({ identity: { attemptId: index }, start: { memoryAvailable: { value: 12345 } }, kernel: { lines: ["KERNEL-DETAIL-SENTINEL"] } })); + const native = path.join(dir, "native.json"); copyFileSync("test/repo/fixtures/test-runner/playwright-retry.json", native); + return { status: index === 0 ? "failed" : "passed", startedAt: start, completedAt: end, shardIndices: [1], outputDir: dir, shards: [{ shardIndex: 1, status: index === 0 ? "infra-failed" : "passed", resourceDiagnosticsPaths: [diagnostic], outputDir: dir, tasks: [{ surfaceId: "e2e:fixture", runner: "playwright", status: "passed", exitCode: 0, durationMs: 25, nativeResultPath: native, resourceDiagnosticsPaths: [diagnostic, path.join(dir, "missing.json")] }] }] } as any; + }); + const combined = combineAttemptReports(attempts, [1], { shardTotal: 1, outputDir: root }); + expect(combined.status).toBe("passed"); + const history = summarizeInfrastructureAttempts(attempts) as any[]; + expect(history[0].shards[0].resourceDiagnosticsPaths).toEqual([path.join(root, "attempt-0/resources.json")]); + const normalized = normalizeProviderShardSurfaces({ provider: "blaxel", providerSummary: combined, outputDir: root, surfaces, delegatedCommand: ["coordinator"], exitCode: 0, coordinatorLogPath: path.join(root, "coordinator.log") }); + expect(normalized).toHaveLength(1); + expect(normalized[0].counts.testsFlakyRecovered).toBe(1); + expect(normalized[0].failedTests).toEqual([]); + const { results, summary } = buildReports({ runId: "resource-run", provider: "blaxel", runDir: root, selection: { surfaces: ["e2e:fixture"] }, surfaces: normalized, startedAt: start, completedAt: end }); + expect(results.surfaces[0].resourceDiagnosticsPaths).toEqual(["attempt-0/resources.json", "attempt-1/resources.json"]); + for (const link of results.surfaces[0].resourceDiagnosticsPaths as string[]) expect(existsSync(path.join(root, link))).toBe(true); + expect(summary.run.status).toBe("passed"); + const events = serializeTimingEvents(buildTimingEvents({ run: { id: "resource-run", commandProvider: "blaxel", timingProvider: "blaxel-playwright-x64-4vcpu", status: "passed", startedAt: start, completedAt: end, durationMs: 1000, selection: { surfaces: ["e2e:fixture"] }, git: { commit: null, tree: null, dirty: false }, environment: { ci: true, os: "linux", arch: "x64", node: process.version }, phases: [] }, shards: [{ index: 1, total: 1, status: "passed", phases: [], surfaces: normalized as any }] })); + for (const compact of [JSON.stringify(summary), JSON.stringify(compactProviderSummary(combined)), events]) { + expect(compact).not.toContain("KERNEL-DETAIL-SENTINEL"); + expect(compact).not.toContain("memoryAvailable"); + } + expect(readFileSync(path.join(root, "attempt-0/resources.json"), "utf8")).toContain("KERNEL-DETAIL-SENTINEL"); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test("baseline-only provider evidence cannot manufacture a selected-surface outcome", () => { + const root = mkdtempSync(path.join(os.tmpdir(), "tv-resource-missing-")); + try { + const diagnostic = path.join(root, "baseline.json"); writeFileSync(diagnostic, JSON.stringify({ end: { status: "missing" } })); + const normalized = normalizeProviderShardSurfaces({ provider: "blaxel", providerSummary: { status: "failed", shards: [{ shardIndex: 1, status: "infra-failed", resourceDiagnosticsPaths: [diagnostic], tasks: [] }] }, outputDir: root, surfaces, delegatedCommand: ["coordinator"], exitCode: 1, coordinatorLogPath: path.join(root, "coordinator.log") }); + const { summary, results } = buildReports({ runId: "missing-native", provider: "blaxel", runDir: root, selection: { surfaces: ["e2e:fixture"] }, surfaces: normalized, startedAt: start, completedAt: end }); + expect(summary.run).toMatchObject({ status: "incomplete", testStatus: "unknown" }); + expect(summary.counts).toMatchObject({ surfacesRun: 0, testsFailed: 0 }); + expect(results.surfaces.map((surface) => surface.id)).toEqual(["provider:blaxel"]); + expect(results.surfaces[0].resourceDiagnosticsPaths).toEqual(["baseline.json"]); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/test/repo/resource-diagnostics.test.ts b/test/repo/resource-diagnostics.test.ts new file mode 100644 index 00000000..7e5426eb --- /dev/null +++ b/test/repo/resource-diagnostics.test.ts @@ -0,0 +1,85 @@ +import { copyFileSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { startResourceDiagnostics, writeResourceDiagnostic } from "../../scripts/test/resource-diagnostics.mjs"; + +const roots: string[] = []; +const identity = { runId: "run-one", shardIndex: 1, attemptId: "lock-one", surfaceId: "e2e:fixture" }; +const startTime = "2026-10-27T00:00:00.000Z"; +const endTime = "2026-10-27T00:00:01.000Z"; +const secret = "credential-query-body-environment-argument-SENTINEL"; +const temporary = () => { const dir = mkdtempSync(path.join(os.tmpdir(), "tv-resource-contract-")); roots.push(dir); return dir; }; +const read = (file: string | null) => JSON.parse(readFileSync(file!, "utf8")); +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +test("retains a baseline, honest units and reset evidence, and kernel evidence for a recovered native attempt", () => { + const root = temporary(); + const nativeResultPath = path.join(root, "native.json"); + copyFileSync("test/repo/fixtures/test-runner/playwright-retry.json", nativeResultPath); + let ended = false; + const probes: Array<[string, unknown]> = []; + const collector = startResourceDiagnostics({ + root, identity: { ...identity, env: secret, args: [secret] }, now: () => new Date(ended ? endTime : startTime), + readProbe: (name, interval) => { + probes.push([name, interval]); + if (name === "temporaryFree") throw Object.assign(new Error(secret), { code: "EACCES" }); + if (name === "oomKills") return ended ? 2 : 10; + if (name === "kernelOOM") return { lines: [`2026-10-27T00:00:00,500000+00:00 Out of memory: Killed process 23 (node) total-vm:4096kB ${secret}`] }; + return 4096; + }, + warn: () => {}, + }); + expect(read(collector.file)).toMatchObject({ identity, start: { observedAt: startTime, memoryAvailable: { value: 4096, unit: "bytes", source: "/proc/meminfo:MemAvailable" }, temporaryFree: { value: null, reason: "EACCES" } }, end: { status: "missing" } }); + ended = true; + expect(collector.finish({ runner: "playwright", nativeResultPath, exitCode: 0 })).toEqual([collector.file]); + const record = read(collector.file); + expect(record).toMatchObject({ end: { observedAt: endTime }, oomDelta: { value: null, reason: "counter-reset" }, nativeEvidence: { status: "complete", failedAttempt: true, recovered: true }, kernel: { status: "available" } }); + expect(probes.find(([name]) => name === "kernelOOM")?.[1]).toMatchObject({ since: startTime, until: endTime }); + expect(record.kernel.lines.join("\n")).toContain("Killed process 23 (node)"); + expect(JSON.stringify(record)).not.toContain(secret); +}); + +test("requests kernel evidence for missing output and preserves unavailable probes without zeroes", () => { + const root = temporary(); + const warnings: string[] = []; + const collector = startResourceDiagnostics({ root, identity, warn: (message) => warnings.push(message), readProbe: () => { throw Object.assign(new Error(secret), { code: "EPERM" }); } }); + collector.finish({ runner: "vitest", nativeResultPath: path.join(root, "absent.json"), exitCode: 17 }); + const record = read(collector.file); + expect(record).toMatchObject({ nativeEvidence: { status: "missing", failedAttempt: null }, start: { oomKills: { value: null, unit: "count", reason: "EPERM" } }, end: { memoryAvailable: { value: null } }, kernel: { status: "unavailable", reason: "EPERM" }, oomDelta: { value: null, reason: "unavailable" } }); + expect(warnings.length).toBeGreaterThan(0); + expect(warnings.join("\n")).not.toContain(secret); +}); + +test("bounds sidecars and the attempt including receipts, reserving compact observations before excerpts", () => { + const root = temporary(); + const warnings: string[] = []; + const native = path.join(root, "native.json"); + copyFileSync("test/repo/fixtures/test-runner/playwright-retry.json", native); + const collector = startResourceDiagnostics({ root, identity, warn: (message) => warnings.push(message), readProbe: (name) => name === "kernelOOM" ? { lines: Array.from({ length: 8000 }, () => "Out of memory: Killed process 23 (node) total-vm:4096kB")} : 1 }); + collector.finish({ runner: "playwright", nativeResultPath: native, exitCode: 0 }); + const record = read(collector.file); + expect(record.start.memoryAvailable.value).toBe(1); + expect(record.end.memoryAvailable.value).toBe(1); + expect(record.kernel.truncated).toBe(true); + expect(Buffer.byteLength(record.kernel.lines.join("\n"))).toBeLessThanOrEqual(64 * 1024); + expect(statSync(collector.file!).size).toBeLessThanOrEqual(128 * 1024); + const receipt = { kind: "recovery", identity, previous: { runId: "old", lockId: "old-lock", worker: "private-worker", lastRenewal: startTime, recoverAfter: endTime, args: secret }, discardedReports: true, cleanup: "confirmed", processes: Array.from({ length: 3000 }, (_, i) => ({ pid: i + 1, startTime: "123", command: "node", outcome: "terminated", args: secret })), env: secret }; + for (let i = 0; i < 80; i++) writeResourceDiagnostic(root, `receipt-${i}`, receipt, { warn: (message) => warnings.push(message) }); + const files = readdirSync(path.join(root, "resources")).map((name) => path.join(root, "resources", name)); + expect(files.reduce((sum, file) => sum + statSync(file).size, 0)).toBeLessThanOrEqual(8 * 1024 * 1024); + expect(files.every((file) => statSync(file).size <= 128 * 1024)).toBe(true); + expect(files.every((file) => !readFileSync(file, "utf8").includes(secret))).toBe(true); + expect(read(collector.file).end.memoryAvailable.value).toBe(1); + expect(warnings.length).toBeGreaterThan(0); +}); + +test("diagnostic write failure warns and returns no unretained link without throwing over a native outcome", () => { + const root = temporary(); + const warnings: string[] = []; + const collector = startResourceDiagnostics({ root, identity, readProbe: () => 1, write: () => { throw Object.assign(new Error(secret), { code: "ENOSPC" }); }, warn: (message) => warnings.push(message) }); + expect(collector.finish({ runner: "vitest", nativeResultPath: null, exitCode: 17 })).toEqual([]); + expect(collector.file === null || !existsSync(collector.file)).toBe(true); + expect(warnings.join("\n")).toContain("ENOSPC"); + expect(warnings.join("\n")).not.toContain(secret); +}); diff --git a/test/repo/test-runner-resource-parent.test.ts b/test/repo/test-runner-resource-parent.test.ts new file mode 100644 index 00000000..1cf2bebe --- /dev/null +++ b/test/repo/test-runner-resource-parent.test.ts @@ -0,0 +1,96 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { expect, test } from "vitest"; +import { createShardPlan } from "../../scripts/test/shard-plan.mjs"; +import { serializeTimingBaseline } from "../../scripts/test/timing-baseline.mjs"; +import { spawnOwnedProcess } from "../helpers/owned-process.ts"; + +const fixtureRoot = "test/fixtures/resource-diagnostics"; +const sourceFile = `${fixtureRoot}/native.fixture.ts`; +const provider = "blaxel-playwright-x64-4vcpu"; +const read = (file: string) => JSON.parse(readFileSync(file, "utf8")); + +for (const entry of ["planned", "targeted"] as const) { + test(`${entry} parent retains real Linux observations around preparation, recovered native attempts and teardown`, { timeout: 30_000 }, () => { + const fixture = createFixture(entry, "retry"); + try { + const result = spawnSync(process.execPath, fixture.args, { encoding: "utf8", env: fixture.env, timeout: 25_000 }); + expect(result.status, result.stderr).toBe(0); + const prepared = read(path.join(fixture.dir, "preparation.json")); + expect(prepared.records).toHaveLength(1); + expect(prepared.records[0].end.status).toBe("missing"); + const record = readDiagnostics(fixture.dir)[0]; + expect(record.identity).toMatchObject({ runId: "resource-parent-run", attemptId: "resource-parent-attempt", shardIndex: 1, surfaceId: "fixture:resources" }); + expect(Date.parse(record.start.observedAt)).toBeLessThanOrEqual(Date.parse(prepared.observedAt)); + expect(Date.parse(record.end.observedAt)).toBeGreaterThanOrEqual(Date.parse(read(path.join(fixture.dir, "teardown.json")).observedAt)); + expect(record.start.memoryAvailable).toMatchObject({ unit: "bytes", source: "/proc/meminfo:MemAvailable" }); + expect(record.start.sharedMemoryUsed).toMatchObject({ unit: "bytes", source: "/proc/meminfo:Shmem" }); + expect(record.start.temporaryFree).toMatchObject({ unit: "bytes", source: "statfs", path: fixture.dir }); + expect(record.start.oomKills).toMatchObject({ unit: "count", source: "/proc/vmstat:oom_kill" }); + for (const key of ["memoryAvailable", "sharedMemoryUsed", "temporaryFree", "oomKills"]) { + const probe = record.start[key]; + if (probe.value === null) expect(probe.reason).toEqual(expect.any(String)); + else expect(probe.value).toBeGreaterThanOrEqual(0); + } + expect(record.nativeEvidence).toMatchObject({ status: "complete", failedAttempt: true, recovered: true }); + expect(["available", "unavailable"]).toContain(record.kernel.status); + if (record.kernel.status === "unavailable") expect(record.kernel.reason).toEqual(expect.any(String)); + const resultRecord = entry === "planned" ? read(path.join(fixture.dir, "results/shard-1.json")).tasks[0] : read(path.join(fixture.dir, "results/lifecycle.json")); + expect(resultRecord.resourceDiagnosticsPaths).toHaveLength(1); + expect(existsSync(resultRecord.resourceDiagnosticsPaths[0])).toBe(true); + expect(resultRecord.processLeaks).toEqual([]); + const attemptPath = path.resolve(path.dirname(resultRecord.resourceDiagnosticsPaths[0]), record.nativeEvidence.attemptResultPath); + const attempts = readFileSync(attemptPath, "utf8").trim().split("\n").map((line) => JSON.parse(line)); + expect(attempts.map((row: any) => row.status)).toEqual(["failed", "passed"]); + } finally { fixture.cleanup(); } + }); +} + +test("targeted parent preserves a baseline after the installed native runner exits before its report", { timeout: 20_000 }, () => { + const fixture = createFixture("targeted", "exit"); + try { + const result = spawnSync(process.execPath, fixture.args, { encoding: "utf8", env: fixture.env, timeout: 15_000 }); + expect(result.status, result.stderr).toBe(17); + expect(read(path.join(fixture.dir, "preparation.json")).records).toHaveLength(1); + const record = readDiagnostics(fixture.dir)[0]; + expect(record).toMatchObject({ end: { status: "observed" }, nativeEvidence: { status: "missing" } }); + expect(existsSync(path.join(fixture.dir, "results/native.json"))).toBe(false); + } finally { fixture.cleanup(); } +}); + +test("interrupted surface parent leaves an explicit missing final observation", { timeout: 20_000 }, async () => { + const fixture = createFixture("targeted", "interrupt"); + const child = spawnOwnedProcess(process.execPath, fixture.args, { env: fixture.env, stdio: "ignore" }); + try { + await expect.poll(() => existsSync(path.join(fixture.dir, "preparation.json")), { timeout: 10_000, interval: 20 }).toBe(true); + expect((await child.dispose()).outcome).not.toBe("survived"); + const records = readDiagnostics(fixture.dir); + expect(records).toHaveLength(1); + expect(records[0]).toMatchObject({ start: { status: "observed" }, end: { status: "missing", reason: "parent-did-not-finalize" } }); + } finally { await child.dispose(); fixture.cleanup(); } +}); + +function readDiagnostics(dir: string) { + const root = path.join(dir, "results/resources"); + return existsSync(root) ? readdirSync(root).filter((name) => name.endsWith(".json")).map((name) => read(path.join(root, name))) : []; +} + +function createFixture(entry: "planned" | "targeted", mode: string) { + mkdirSync(".test-runs", { recursive: true }); + const dir = mkdtempSync(path.join(process.cwd(), ".test-runs/resource-parent-")); + const results = path.join(dir, "results"); + mkdirSync(results); + const nativeConfig = path.join(dir, "native.config.mjs"); + writeFileSync(nativeConfig, `export default ${JSON.stringify({ test: { include: [path.resolve(sourceFile)], globalSetup: [path.resolve(`${fixtureRoot}/setup.mjs`)], pool: "forks", maxWorkers: 1, retry: 2 } })};\n`); + const preCommand = [process.execPath, path.resolve(`${fixtureRoot}/prepare.mjs`)]; + const surface = { id: "fixture:resources", runner: "vitest", kind: "e2e", package: null, config: nativeConfig, absoluteConfig: nativeConfig, roots: [sourceFile], excludeRoots: [], cwd: ".", supports: ["file"], preflight: [], tags: [], agent: false, preCommand, command: null, executionGroup: { id: "fixture:resources", name: "resource fixture", order: 1 } }; + const registryPath = path.join(dir, "registry.mjs"); + writeFileSync(registryPath, `export default ${JSON.stringify({ suites: { all: { include: ["kind:e2e"] } }, surfaces: [surface] })};\n`); + const plan = createShardPlan({ inventory: [{ path: sourceFile, surfaceId: surface.id, runner: "vitest" }], surfaces: [surface] as any, baseline: { schemaVersion: 1, sourceThrough: null, files: {} }, baselineBytes: serializeTimingBaseline({ schemaVersion: 1, sourceThrough: null, files: {} }), timingProvider: provider, testedCommit: execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(), testedTree: execFileSync("git", ["rev-parse", "HEAD^{tree}"], { encoding: "utf8" }).trim(), shardTotal: 1 }); + const planPath = path.join(dir, "plan.json"); + writeFileSync(planPath, JSON.stringify(plan)); + const env = { ...process.env, GITHUB_ACTIONS: "false", TV_TEST_RUNNER_SELFTEST: "1", TEST_SHARD_CONFIG_MODULE: registryPath, TV_TEST_TIMING_PROVIDER: provider, TV_TEST_RUN_ID: "resource-parent-run", TV_TEST_SHARD_INDEX: "1", TV_TEST_SHARD_ATTEMPT: "resource-parent-attempt", RESOURCE_FIXTURE_ROOT: dir, RESOURCE_FIXTURE_MODE: mode, TMPDIR: dir, TV_TARGET_RUNNER: "vitest", TV_TARGET_CONFIG: nativeConfig, TV_TARGET_NATIVE_RESULT: path.join(results, "native.json"), TV_VITEST_ATTEMPT_FILE: path.join(results, "native.attempts.ndjson") }; + const args = entry === "planned" ? ["scripts/run-test-shard.mjs", "--plan", planPath, "--shard", "1", "--results-dir", results, "--test-retries", "2"] : ["scripts/test/supervised-command.mjs", "--run-id", "resource-parent-run", "--surface", surface.id, "--result", path.join(results, "lifecycle.json"), "--pre-command-json", JSON.stringify(preCommand), "--", "bash", "-c", 'exec node node_modules/vitest/vitest.mjs run --config "${TV_TARGET_VITEST_CONFIG:-$TV_TARGET_CONFIG}" --reporter=json --outputFile "$TV_TARGET_NATIVE_RESULT"' ]; + return { dir, args, env, cleanup: () => rmSync(dir, { recursive: true, force: true }) }; +} From e4a094a7275669987545d281ebfa81dc656b081a Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:59:02 +0000 Subject: [PATCH 21/64] Require diagnostics for each combined Vitest member --- test/repo/test-runner-shard-worker.test.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test/repo/test-runner-shard-worker.test.ts b/test/repo/test-runner-shard-worker.test.ts index 93e948e5..c4cfe5c7 100644 --- a/test/repo/test-runner-shard-worker.test.ts +++ b/test/repo/test-runner-shard-worker.test.ts @@ -37,9 +37,9 @@ describe("planned shard worker", () => { const fixture = workerFixture(["fixture:vitest-one", "fixture:vitest-two"], [ inventory("vitest-one.fixture.ts", "fixture:vitest-one", "vitest"), inventory("vitest-two.fixture.ts", "fixture:vitest-two", "vitest"), - ]); + ], "blaxel-playwright-x64-4vcpu"); try { - const result = runWorker(fixture, { TV_SHARD_RETRY_FIXTURE: "1" }); + const result = runWorker(fixture, { TV_SHARD_RETRY_FIXTURE: "1", TV_TEST_RUN_ID: "combined-resource-fixture", TV_TEST_SHARD_ATTEMPT: "combined-attempt", TV_TEST_SHARD_INDEX: "1" }); expect(result.status, result.stderr).toBe(0); const summary = shardSummary(fixture); expect(summary.preflights).toEqual([ @@ -73,6 +73,17 @@ describe("planned shard worker", () => { }); expect(normalized.find((surface) => surface.id === "fixture:vitest-one")?.counts.testsFlakyRecovered).toBe(0); expect(normalized.find((surface) => surface.id === "fixture:vitest-two")?.counts.testsFlakyRecovered).toBe(1); + for (const [index, task] of summary.tasks.entries()) { + expect(task.resourceDiagnosticsPaths).toHaveLength(1); + const diagnosticPath = task.resourceDiagnosticsPaths[0]; + const diagnostic = JSON.parse(readFileSync(diagnosticPath, "utf8")); + expect(diagnostic).toMatchObject({ + identity: { runId: "combined-resource-fixture", attemptId: "combined-attempt", shardIndex: 1, surfaceId: task.surfaceId }, + start: { status: "observed" }, end: { status: "observed" }, + nativeEvidence: { status: "complete", failedAttempt: index === 1, recovered: index === 1 }, + }); + expect(path.resolve(path.dirname(diagnosticPath), diagnostic.nativeEvidence.resultPath)).toBe(task.nativeResultPath); + } } finally { fixture.cleanup(); } }); From 24229bd3efa85915ef85f36bc01c1a49be8d7277 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:59:33 +0000 Subject: [PATCH 22/64] Retain bounded resource evidence through native failures and retries --- scripts/run-blaxel-testshards.mjs | 51 +++++- scripts/test/planned-shard-worker.mjs | 20 ++- scripts/test/provider-normalization.mjs | 6 +- scripts/test/provider-outputs.d.mts | 2 + scripts/test/provider-outputs.mjs | 11 +- scripts/test/reporting.d.mts | 4 +- scripts/test/reporting.mjs | 10 +- scripts/test/resource-diagnostics.d.mts | 4 + scripts/test/resource-diagnostics.mjs | 222 ++++++++++++++++++++++-- scripts/test/supervised-command.mjs | 21 ++- 10 files changed, 327 insertions(+), 24 deletions(-) diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index c0d6dfbe..c77df60c 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -10,6 +10,7 @@ import { blaxelCheckoutScript, blaxelDependencyScript, resolveBlaxelRepositoryUr import { transferShardInputs } from "./test/blaxel-shard-dispatch.mjs"; import { buildLinuxStaleOwnerAuditInvocation, parseLinuxStaleOwnerAuditOutput } from "./test/blaxel-stale-owner-audit.mjs"; import { normalizeSetupTimingStatus, normalizeTimingPhase } from "./test/phase-metrics.mjs"; +import { RESOURCE_SIDECAR_BYTES, RESOURCE_ATTEMPT_BYTES } from "./test/resource-diagnostics.mjs"; import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; @@ -438,6 +439,8 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE const command = `bash ${remoteScript}`; let stream; let retainedResultProgress = null; + let resourceDiagnosticsPaths = []; + const copiedArtifacts = new Map(); let failureStep = "worker-execution"; try { lease.executionRequested = true; @@ -484,6 +487,7 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE summaryPath: fs.existsSync(summaryPath) ? summaryPath : null, summaryError: shardSummaryError, nativeResultPath, + resourceDiagnosticsPaths: [...resourceDiagnosticsPaths], logPath: fs.existsSync(logPath) ? logPath : null, setupTimings, planId: shardSummary?.planId ?? shardPlan?.planId, @@ -525,9 +529,11 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE } } catch {} retainResultProgress(); - const copiedArtifacts = new Map(); + resourceDiagnosticsPaths = await copyResourceDiagnostics(sandbox, { outputDir, shardIndex, lockId, copiedArtifacts }); + retainResultProgress(); for (const [taskIndex, task] of (shardSummary?.tasks ?? []).entries()) { const copiedTask = { ...tasks[taskIndex] }; + copiedTask.resourceDiagnosticsPaths = (task.resourceDiagnosticsPaths ?? []).map((remotePath) => copiedArtifacts.get(remotePath)).filter(Boolean); for (const field of ["nativeResultPath", "attemptResultPath", "workspaceNativeResultPath", "lifecycleResultPath", "processLeakResultPath"]) { if (!task[field]) continue; copiedTask[field] = await copyRemoteArtifact(sandbox, task[field], { outputDir, shardIndex, copiedArtifacts }); @@ -569,15 +575,17 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE failureStep: hasSummaryFile ? "summary-parse" : "summary-read", failureMessage: shardSummaryError ?? `Shard ${shardIndex}/${shardTotal} produced no readable summary.`, } : null; - return { shardIndex, shardTotal, shard: `${shardIndex}/${shardTotal}`, sandbox: sandbox.metadata.name, outputDir, status, exitCode, startedAt: new Date(overallStarted).toISOString(), completedAt, durationMs: Date.now() - overallStarted, summaryPath: fs.existsSync(summaryPath) ? summaryPath : null, summaryError: shardSummaryError, nativeResultPath, logPath, setupTimings, planId: shardSummary?.planId ?? shardPlan?.planId, timingProvider: shardSummary?.timingProvider ?? shardPlan?.timingProvider, testedTree: shardSummary?.testedTree ?? shardPlan?.testedTree, testedCommit: shardSummary?.testedCommit ?? shardPlan?.testedCommit, predictedDurationMs: shardSummary?.predictedDurationMs, phases: [...(shardSummary?.phases ?? []), reportPhase], assignedFiles: shardSummary?.assignedFiles ?? [], collectedFiles: shardSummary?.collectedFiles ?? [], failureKind: identityError ? "plan-validation" : resultError ? "result-report" : missingSummaryFailure?.failureKind ?? shardSummary?.failureKind ?? failedInfrastructureTask?.failureKind, failureStep: identityError ? "plan-validation" : resultError ? "summary-validation" : missingSummaryFailure?.failureStep ?? shardSummary?.failureStep ?? failedInfrastructureTask?.failureStep, failureMessage: identityError ?? resultError ?? missingSummaryFailure?.failureMessage ?? shardSummary?.failureMessage ?? failedInfrastructureTask?.failureMessage, processLeaks: shardSummary?.processLeaks ?? [], tasks }; + return { shardIndex, shardTotal, shard: `${shardIndex}/${shardTotal}`, sandbox: sandbox.metadata.name, outputDir, status, exitCode, startedAt: new Date(overallStarted).toISOString(), completedAt, durationMs: Date.now() - overallStarted, summaryPath: fs.existsSync(summaryPath) ? summaryPath : null, summaryError: shardSummaryError, nativeResultPath, resourceDiagnosticsPaths, logPath, setupTimings, planId: shardSummary?.planId ?? shardPlan?.planId, timingProvider: shardSummary?.timingProvider ?? shardPlan?.timingProvider, testedTree: shardSummary?.testedTree ?? shardPlan?.testedTree, testedCommit: shardSummary?.testedCommit ?? shardPlan?.testedCommit, predictedDurationMs: shardSummary?.predictedDurationMs, phases: [...(shardSummary?.phases ?? []), reportPhase], assignedFiles: shardSummary?.assignedFiles ?? [], collectedFiles: shardSummary?.collectedFiles ?? [], failureKind: identityError ? "plan-validation" : resultError ? "result-report" : missingSummaryFailure?.failureKind ?? shardSummary?.failureKind ?? failedInfrastructureTask?.failureKind, failureStep: identityError ? "plan-validation" : resultError ? "summary-validation" : missingSummaryFailure?.failureStep ?? shardSummary?.failureStep ?? failedInfrastructureTask?.failureStep, failureMessage: identityError ?? resultError ?? missingSummaryFailure?.failureMessage ?? shardSummary?.failureMessage ?? failedInfrastructureTask?.failureMessage, processLeaks: shardSummary?.processLeaks ?? [], tasks }; } catch (error) { stream?.close?.(); + resourceDiagnosticsPaths = await copyResourceDiagnostics(sandbox, { outputDir, shardIndex, lockId, copiedArtifacts }); appendLog(logPath, `\n[coordinator] ${error.stack || error.message}\n`); const completedAt = new Date().toISOString(); const plannedShard = shardPlan?.shards?.[shardIndex - 1]; const failure = classifyBlaxelCoordinatorFailure({ failureStep, message: error.message }); return { ...retainedResultProgress, + resourceDiagnosticsPaths, shardIndex, shardTotal, shard: `${shardIndex}/${shardTotal}`, @@ -647,6 +655,37 @@ function retainTaskWithoutRemoteArtifacts(task) { if (retained[field]) retained[field] = null; } if (Array.isArray(retained.generatedInputPaths)) retained.generatedInputPaths = []; + retained.resourceDiagnosticsPaths = []; + return retained; +} + +async function copyResourceDiagnostics(sandbox, { outputDir, shardIndex, lockId, copiedArtifacts }) { + const root = "/workspace/television/.testshards/results/resources"; + const retained = []; + let bytes = 0; + try { + const listing = await sandbox.fs.ls(root); + for (const entry of (listing.files ?? []).slice(0, 1024)) { + if (!/^[A-Za-z0-9_.-]+\.json$/.test(entry.name)) continue; + try { + if (entry.size > RESOURCE_SIDECAR_BYTES || bytes + entry.size > RESOURCE_ATTEMPT_BYTES) throw new Error("budget"); + const remotePath = `${root}/${entry.name}`; + const raw = await sandbox.fs.read(remotePath); + const size = Buffer.byteLength(raw); + if (size > RESOURCE_SIDECAR_BYTES || bytes + size > RESOURCE_ATTEMPT_BYTES) throw new Error("budget"); + const record = JSON.parse(raw); + if (record.schemaVersion !== 1 || record.identity?.runId !== runId || record.identity?.shardIndex !== shardIndex || record.identity?.attemptId !== lockId) throw new Error("identity"); + const localPath = path.join(outputDir, "artifacts", `shard-${shardIndex}`, "resources", entry.name); + fs.mkdirSync(path.dirname(localPath), { recursive: true }); + fs.writeFileSync(localPath, raw); + copiedArtifacts.set(remotePath, localPath); + retained.push(localPath); bytes += size; + } catch { console.error(`[resource diagnostics] shard ${shardIndex}: sidecar unavailable, invalid or over budget`); } + } + if ((listing.files?.length ?? 0) > 1024) console.error(`[resource diagnostics] shard ${shardIndex}: file listing truncated`); + } catch (error) { + if (error?.status !== 404 && error?.response?.status !== 404) console.error(`[resource diagnostics] shard ${shardIndex}: directory unavailable`); + } return retained; } @@ -707,7 +746,7 @@ function targetCommand() { if (target.runner) { const args = ["cd", shellQuote(target.cwd), "&&"]; if (target.runner === "vitest") { - args.push("npx", "vitest", "run", "--config", shellQuote(target.config), "--reporter=json", '--outputFile "$TV_TARGET_NATIVE_RESULT"'); + args.push("npx", "vitest", "run", "--config", '"$TV_TARGET_VITEST_CONFIG"', "--reporter=json", '--outputFile "$TV_TARGET_NATIVE_RESULT"'); if (target.file) args.push(shellQuote(target.file)); if (target.grep) args.push("-t", shellQuote(target.grep)); if (target.retries) args.push(`--retry=${target.retries}`); @@ -783,6 +822,9 @@ export npm_config_fund=false export PLAYWRIGHT_BROWSERS_PATH=${playwrightBrowsersPath} export TEST_RETRIES=${testRetries} export TV_TEST_TIMING_PROVIDER=blaxel-playwright-x64-4vcpu +export TV_TEST_RUN_ID=${shellQuote(runId)} +export TV_TEST_SHARD_INDEX=${shardIndex} +export TV_TEST_SHARD_ATTEMPT=${shellQuote(lockId)} mkdir -p /workspace /cache/npm if [ -z "\${GH_TOKEN:-}" ]; then echo "GH_TOKEN missing" >&2; exit 2; fi setup_metrics=/tmp/tv-testshard-setup-${shardIndex}.jsonl @@ -934,6 +976,7 @@ elif [ ${shellQuote(suite)} = target ]; then export TV_TARGET_RUN_ID=${shellQuote(`${runId}:${shardIndex}`)} export TV_TARGET_NAME=${shellQuote(`${targetOptions()?.workspace ?? "target"} ${targetOptions()?.file ?? ""} ${targetOptions()?.grep ?? ""}`)} export TV_TARGET_RUNNER=${shellQuote(targetOptions()?.runner ?? "target")} + export TV_TARGET_CONFIG=${shellQuote(targetOptions()?.config ? path.posix.join("/workspace/television", targetOptions().config) : "")} export TV_TARGET_COMMAND=${shellQuote(targetCommand())} export TV_TARGET_LIFECYCLE_RESULT="/workspace/television/.testshards/results/target-lifecycle-${shardIndex}.json" export TV_TARGET_NATIVE_RESULT="/workspace/television/.testshards/results/native-shard-${shardIndex}.json" @@ -970,6 +1013,8 @@ const task = { durationMs: Date.now() - Date.parse(startedAt), nativeResultPath: fs.existsSync(process.env.TV_TARGET_NATIVE_RESULT) ? process.env.TV_TARGET_NATIVE_RESULT : null, lifecycleResultPath: process.env.TV_TARGET_LIFECYCLE_RESULT, + attemptResultPath: lifecycle.attemptResultPath || null, + resourceDiagnosticsPaths: lifecycle.resourceDiagnosticsPaths || [], processLeaks, failureKind: lifecycle.failureKind || null, failureStep: lifecycle.failureStep || null, diff --git a/scripts/test/planned-shard-worker.mjs b/scripts/test/planned-shard-worker.mjs index cf536201..9fba1c64 100644 --- a/scripts/test/planned-shard-worker.mjs +++ b/scripts/test/planned-shard-worker.mjs @@ -14,6 +14,7 @@ import { runnerEnv } from "./runner-env.mjs"; import { createSurfaceSupervisor } from "./surface-supervisor.mjs"; import { startSurfaceServices } from "./surface-services.mjs"; import { reapStaleOwnerProcesses } from "./stale-owner-reaper.mjs"; +import { surfaceResourceDiagnostics } from "./resource-diagnostics.mjs"; export async function runPlannedShard(rawOptions = {}, env = process.env) { const planPath = path.resolve(rawOptions.plan ?? env.TEST_SHARD_PLAN ?? ""); @@ -179,6 +180,7 @@ function relocateTaskArtifacts(summary, resultsDir, providerOutputDir, nativeOut workspaceNativeResultPath: relocateNative(task.workspaceNativeResultPath), lifecycleResultPath: relocateProvider(task.lifecycleResultPath), processLeakResultPath: relocateProvider(task.processLeakResultPath), + resourceDiagnosticsPaths: (task.resourceDiagnosticsPaths ?? []).map(relocateProvider), generatedInputPaths: (task.generatedInputPaths ?? []).map(relocateProvider), })), }; @@ -208,10 +210,12 @@ async function runVitestWorkspace(tasks, { resultsDir, shardIndex, skipPreComman const nativeResultPath = path.join(resultsDir, `native-unit-workspaces-shard-${shardIndex}.json`); const attemptResultPath = path.join(resultsDir, `native-unit-workspaces-shard-${shardIndex}.attempts.ndjson`); const supervisor = createSurfaceSupervisor({ runId: lifecycleRunId, surfaceIds: tasks.map((task) => task.surfaceId) }); + const diagnostics = tasks.map((task) => surfaceResourceDiagnostics({ resultsDir, surfaceId: task.surfaceId, env, runId: lifecycleRunId })); let pre; let exitCode = 0; let processLeaks = []; let commandStarted = null; + let splitStatuses = new Map(); fs.rmSync(attemptResultPath, { force: true }); const commandArgs = vitestProjectsArgs({ configPath: workspacePath, configPaths: tasks.map((task) => path.resolve(task.config)) }); try { @@ -225,10 +229,16 @@ async function runVitestWorkspace(tasks, { resultsDir, shardIndex, skipPreComman exitCode = pre.exitCode; } } finally { - processLeaks = await supervisor.finish(); + let cleanupConfirmed = false; + try { processLeaks = await supervisor.finish(); cleanupConfirmed = true; } + finally { + try { splitStatuses = splitVitestWorkspaceResult({ nativeResultPath, tasks }); } + finally { + for (const [index, task] of tasks.entries()) task.resourceDiagnosticsPaths = diagnostics[index].finish({ runner: "vitest", nativeResultPath: task.nativeResultPath, attemptResultPath, exitCode, incomplete: !cleanupConfirmed || !pre || pre.failed || processLeaks.length > 0 }); + } + } } if (pre.failed) return tasks.map((task) => failedTask(task, pre.exitCode, started, { preCommandsSkipped: skipPreCommands, failureKind: pre.failureKind, failureStep: "pre-command", failureMessage: pre.failureMessage, phases: pre.phases, generatedInputPaths: [workspacePath], processLeaks })); - const splitStatuses = splitVitestWorkspaceResult({ nativeResultPath, tasks }); return tasks.map((task) => { const status = exitCode === 0 ? 0 : (splitStatuses.get(task.surfaceId) ?? exitCode); const collected = collectedVitestFiles(task.nativeResultPath, task); @@ -239,6 +249,7 @@ async function runVitestWorkspace(tasks, { resultsDir, shardIndex, skipPreComman async function runStandalone(task, { resultsDir, shardIndex, skipPreCommands, env, lifecycleRunId }) { const started = Date.now(); const supervisor = createSurfaceSupervisor({ runId: lifecycleRunId, surfaceIds: [task.surfaceId] }); + const diagnostics = surfaceResourceDiagnostics({ resultsDir, surfaceId: task.surfaceId, env, runId: lifecycleRunId }); let pre; let command; let args; @@ -300,7 +311,9 @@ async function runStandalone(task, { resultsDir, shardIndex, skipPreCommands, en pre?.phases.push(phase("surface-services-stop", serviceStopped, 1)); } } - processLeaks = await supervisor.finish(); + let cleanupConfirmed = false; + try { processLeaks = await supervisor.finish(); cleanupConfirmed = true; } + finally { task.resourceDiagnosticsPaths = diagnostics.finish({ runner: task.runner, nativeResultPath: task.nativeResultPath, attemptResultPath, exitCode, incomplete: !cleanupConfirmed || !pre || pre.failed || Boolean(serviceFailure) || processLeaks.length > 0 }); } } if (pre.failed) return failedTask(task, pre.exitCode, started, { preCommandsSkipped: skipPreCommands, failureKind: pre.failureKind, failureStep: "pre-command", failureMessage: pre.failureMessage, phases: pre.phases, processLeaks }); const collected = task.runner === "vitest" ? collectedVitestFiles(task.nativeResultPath, task) : collectedPlaywrightFiles(task.nativeResultPath, task); @@ -366,6 +379,7 @@ function resultForTask(task, exitCode, started, extras) { attemptResultPath: extras.attemptResultPath && fs.existsSync(extras.attemptResultPath) ? extras.attemptResultPath : null, workspaceNativeResultPath: extras.workspaceNativeResultPath ?? null, generatedInputPaths: extras.generatedInputPaths ?? [], + resourceDiagnosticsPaths: task.resourceDiagnosticsPaths ?? [], preCommandsSkipped: extras.preCommandsSkipped, phases: extras.phases ?? [], processLeaks, ...(mismatch diff --git a/scripts/test/provider-normalization.mjs b/scripts/test/provider-normalization.mjs index b2277b56..8b0afe50 100644 --- a/scripts/test/provider-normalization.mjs +++ b/scripts/test/provider-normalization.mjs @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import { normalizeSetupTimingStatus } from "./phase-metrics.mjs"; -import { normalizeSurfaceResult } from "./reporting.mjs"; +import { normalizeSurfaceResult, retainedResourcePaths } from "./reporting.mjs"; export function normalizeProviderShardSurfaces({ provider, providerSummary, outputDir, surfaces, delegatedCommand, exitCode, coordinatorLogPath }) { const bySurface = new Map(surfaces.map((surface) => [surface.id, surface])); @@ -49,6 +49,7 @@ export function normalizeProviderShardSurfaces({ provider, providerSummary, outp durationMs: shard.durationMs ?? providerSummary.durationMs ?? 0, logPath: path.relative(process.cwd(), shard.logPath ?? coordinatorLogPath), nativeResultPath: path.relative(process.cwd(), shard.nativeResultPath), + resourceDiagnosticsPaths: shard.resourceDiagnosticsPaths, })); } } @@ -201,6 +202,7 @@ export function mergeSurfaceParts(parts) { durationSource: "task-wall", nativeResultPath: group.map((part) => part.nativeResultPath).filter(Boolean)[0] ?? null, nativeResultPaths: group.map((part) => part.nativeResultPath).filter(Boolean), + resourceDiagnosticsPaths: retainedResourcePaths(group.flatMap((part) => part.resourceDiagnosticsPaths ?? [])), failureKind: firstInfrastructureFailure?.failureKind ?? first.failureKind, failureStep: firstInfrastructureFailure?.failureStep ?? first.failureStep, failureMessage: firstInfrastructureFailure?.failureMessage ?? first.failureMessage, @@ -248,6 +250,7 @@ function normalizeTaskPart({ task, surface, shard, outputDir, coordinatorLogPath logPath: path.relative(process.cwd(), shard.logPath ?? coordinatorLogPath), nativeResultPath, attemptResultPath, + resourceDiagnosticsPaths: task.resourceDiagnosticsPaths, processLeaks: task.processLeaks ?? [], }); const infraStatus = task.infraStatus === "incomplete" || normalized.infraStatus === "incomplete" ? "incomplete" : "completed"; @@ -357,5 +360,6 @@ function providerFailureSurfaces({ provider, failures, delegatedCommand, coordin infraFailedShards: failure.status === "infra-failed" && failure.shardIndex ? [failure.shardIndex] : [], logPath: retainedLogPath(failure.logPath, coordinatorLogPath), nativeResultPath: null, + resourceDiagnosticsPaths: retainedResourcePaths(failure.resourceDiagnosticsPaths), })); } diff --git a/scripts/test/provider-outputs.d.mts b/scripts/test/provider-outputs.d.mts index c8e2809a..75625c42 100644 --- a/scripts/test/provider-outputs.d.mts +++ b/scripts/test/provider-outputs.d.mts @@ -10,6 +10,7 @@ export interface ProviderTask { exitCode?: number | null; durationMs?: number; nativeResultPath?: string | null; + resourceDiagnosticsPaths?: string[]; [key: string]: unknown; } @@ -28,6 +29,7 @@ export interface ProviderShard { leaseAcquired?: boolean; executionStarted?: boolean; nativeResultPath?: string | null; + resourceDiagnosticsPaths?: string[]; [key: string]: unknown; } diff --git a/scripts/test/provider-outputs.mjs b/scripts/test/provider-outputs.mjs index 2f8d3ff2..c2e0247b 100644 --- a/scripts/test/provider-outputs.mjs +++ b/scripts/test/provider-outputs.mjs @@ -1,6 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import { normalizeSetupTimingStatus, normalizeTimingPhase } from "./phase-metrics.mjs"; +import { retainedResourcePaths } from "./reporting.mjs"; export function providerCommandExitCode({ delegatedExitCode, reportStatus }) { if (delegatedExitCode !== 0) return delegatedExitCode || 1; @@ -63,6 +64,7 @@ export function compactProviderSummary(report, { outputDir = null } = {}) { logPath: shard.logPath, summaryPath: shard.summaryPath, outputDir: shard.outputDir, + resourceDiagnosticsPaths: retainedResourcePaths(shard.resourceDiagnosticsPaths), })), failedTasks, infraFailedShards, @@ -150,6 +152,7 @@ export function summarizeInfrastructureAttempts(reports) { logPath: shard.logPath, summaryPath: shard.summaryPath, outputDir: shard.outputDir, + resourceDiagnosticsPaths: retainedResourcePaths(shard.resourceDiagnosticsPaths), })), })); } @@ -189,7 +192,13 @@ export function combineAttemptReports(attempts, expectedShardIndices, { shardTot message: `No infrastructure attempt returned a result for shard ${shardIndex}/${shardTotal}.`, })); } - const shards = expectedShardIndices.map((shardIndex) => latestByShard.get(shardIndex)).sort((a, b) => a.shardIndex - b.shardIndex); + const shards = expectedShardIndices.map((shardIndex) => ({ ...latestByShard.get(shardIndex) })).sort((a, b) => a.shardIndex - b.shardIndex); + // Retain only links from superseded attempts, never their test outcomes. + for (const shard of shards) { + const history = attempts.flatMap((attempt) => (attempt.shards ?? []).filter((part) => part.shardIndex === shard.shardIndex)); + shard.resourceDiagnosticsPaths = retainedResourcePaths(history.flatMap((part) => [...(part.resourceDiagnosticsPaths ?? []), ...(part.tasks ?? []).flatMap((task) => task.resourceDiagnosticsPaths ?? [])])); + shard.tasks = (shard.tasks ?? []).map((task) => ({ ...task, resourceDiagnosticsPaths: retainedResourcePaths(history.flatMap((part) => (part.tasks ?? []).filter((candidate) => candidate.surfaceId === task.surfaceId).flatMap((candidate) => candidate.resourceDiagnosticsPaths ?? []))) })); + } const last = attempts.at(-1); const failedTasks = failedTasksFromShards(shards, { shardTotal, outputDir }); const startedAt = attempts[0]?.startedAt ?? attempts[0]?.run?.createdAt ?? new Date().toISOString(); diff --git a/scripts/test/reporting.d.mts b/scripts/test/reporting.d.mts index eef8a7d0..c68c7bb9 100644 --- a/scripts/test/reporting.d.mts +++ b/scripts/test/reporting.d.mts @@ -20,6 +20,7 @@ export interface NormalizedSurfaceResult { logPath: string | null; nativeResultPath?: string | null; nativeResultPaths?: string[]; + resourceDiagnosticsPaths?: string[]; counts: Counts; failedTests: Array>; flakyRecoveredTests: Array>; @@ -62,4 +63,5 @@ export function buildReports(args: { startedAt: string; completedAt: string; }): { summary: ReportSummary; results: ReportResults }; -export function normalizeSurfaceResult(args: { surface: Pick & Partial & { durationSource?: NormalizedSurfaceResult["durationSource"] }; command: string[]; exitCode: number | null; durationMs: number; logPath: string; nativeResultPath: string | null; attemptResultPath?: string | null; processLeaks?: Array> }): NormalizedSurfaceResult; +export function normalizeSurfaceResult(args: { surface: Pick & Partial & { durationSource?: NormalizedSurfaceResult["durationSource"] }; command: string[]; exitCode: number | null; durationMs: number; logPath: string; nativeResultPath: string | null; attemptResultPath?: string | null; resourceDiagnosticsPaths?: string[]; processLeaks?: Array> }): NormalizedSurfaceResult; +export function retainedResourcePaths(paths?: string[]): string[]; diff --git a/scripts/test/reporting.mjs b/scripts/test/reporting.mjs index 809b9689..13242288 100644 --- a/scripts/test/reporting.mjs +++ b/scripts/test/reporting.mjs @@ -79,6 +79,7 @@ export function buildReports({ runId, provider = null, commandProvider = provide logPath: surface.logPath, nativeResultPath: Array.isArray(surface.nativeResultPath) ? (surface.nativeResultPath[0] ?? null) : surface.nativeResultPath, nativeResultPaths: surface.nativeResultPaths ?? (Array.isArray(surface.nativeResultPath) ? surface.nativeResultPath : [surface.nativeResultPath].filter(Boolean)), + resourceDiagnosticsPaths: retainedResourcePaths(surface.resourceDiagnosticsPaths).map((file) => path.relative(runDir, file).replaceAll("\\", "/")), failureKind: surface.failureKind ?? null, failureStep: surface.failureStep ?? null, failureMessage: surface.failureMessage ?? null, @@ -135,7 +136,7 @@ function compactObject(value) { return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined && item !== null)); } -export function normalizeSurfaceResult({ surface, command, exitCode, durationMs, logPath, nativeResultPath, attemptResultPath = null, processLeaks = [] }) { +export function normalizeSurfaceResult({ surface, command, exitCode, durationMs, logPath, nativeResultPath, attemptResultPath = null, resourceDiagnosticsPaths = [], processLeaks = [] }) { const nativeReport = readJsonResult(nativeResultPath); const native = nativeReport.value; const base = { @@ -148,6 +149,7 @@ export function normalizeSurfaceResult({ surface, command, exitCode, durationMs, logPath, nativeResultPath, attemptResultPath, + resourceDiagnosticsPaths: retainedResourcePaths(resourceDiagnosticsPaths), infraStatus: "completed", processLeaks: deduplicateProcessLeaks(processLeaks), }; @@ -170,6 +172,12 @@ export function normalizeSurfaceResult({ surface, command, exitCode, durationMs, return { ...base, status: exitCode === 0 ? "passed" : "failed", counts: emptyCounts(), failedTests: [], flakyRecoveredTests: [], files: [] }; } +// Exact paths only: a basename lookup could attach another infrastructure +// attempt's observation when this attempt's download is absent. +export function retainedResourcePaths(paths = []) { + return [...new Set(paths.filter((file) => typeof file === "string" && fs.existsSync(file)).map((file) => path.resolve(file)))].sort(); +} + function normalizePlaywright(native, exitCode, surface) { const tests = []; collectPlaywrightSpecs(native.suites ?? [], tests, [], surface); diff --git a/scripts/test/resource-diagnostics.d.mts b/scripts/test/resource-diagnostics.d.mts index 957d79db..783c9390 100644 --- a/scripts/test/resource-diagnostics.d.mts +++ b/scripts/test/resource-diagnostics.d.mts @@ -1,2 +1,6 @@ export function startResourceDiagnostics(options: { root: string; identity: Record; readProbe?: (name: string, interval?: Record) => unknown; now?: () => Date; write?: (file: string, contents: string) => void; warn?: (message: string) => void }): { file: string | null; finish: (outcome: { runner: string; nativeResultPath: string | null; attemptResultPath?: string | null; exitCode: number; incomplete?: boolean }) => string[] }; export function writeResourceDiagnostic(root: string, name: string, record: Record, options?: { write?: (file: string, contents: string) => void; warn?: (message: string) => void }): string | null; +export const RESOURCE_SIDECAR_BYTES: number; +export const RESOURCE_ATTEMPT_BYTES: number; +export function surfaceResourceDiagnostics(options: { resultsDir: string; surfaceId: string; env?: NodeJS.ProcessEnv; runId: string }): ReturnType; +export function readLinuxResourceProbe(name: string, options?: Record): unknown; diff --git a/scripts/test/resource-diagnostics.mjs b/scripts/test/resource-diagnostics.mjs index 1f42c079..2d50538d 100644 --- a/scripts/test/resource-diagnostics.mjs +++ b/scripts/test/resource-diagnostics.mjs @@ -1,17 +1,217 @@ import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { normalizeSurfaceResult } from "./reporting.mjs"; -// Initial public boundary for the red collector/writer contracts. -export function startResourceDiagnostics({ root, identity }) { - const file = path.join(root, "resources", `${identity.surfaceId.replace(/[^a-zA-Z0-9-]/g, "-")}.json`); - fs.mkdirSync(path.dirname(file), { recursive: true }); - fs.writeFileSync(file, JSON.stringify({ schemaVersion: 1 })); - return { file, finish: () => [file] }; +export const RESOURCE_SIDECAR_BYTES = 128 * 1024; +export const RESOURCE_ATTEMPT_BYTES = 8 * 1024 * 1024; +const KERNEL_BYTES = 64 * 1024; +const COMPACT_RESERVE_BYTES = 8 * 1024; +const probes = { + memoryAvailable: { unit: "bytes", source: "/proc/meminfo:MemAvailable" }, + sharedMemoryUsed: { unit: "bytes", source: "/proc/meminfo:Shmem" }, + temporaryFree: { unit: "bytes", source: "statfs" }, + oomKills: { unit: "count", source: "/proc/vmstat:oom_kill" }, +}; +const warnDefault = (message) => console.error(`[resource diagnostics] ${message}`); + +/** Shared by planned tasks and the targeted surface parent, never the native child. */ +export function surfaceResourceDiagnostics({ resultsDir, surfaceId, env = process.env, runId }) { + if (env.TV_TEST_TIMING_PROVIDER !== "blaxel-playwright-x64-4vcpu") return { file: null, finish: () => [] }; + return startResourceDiagnostics({ root: resultsDir, identity: { + runId: env.TV_TEST_RUN_ID ?? runId, + shardIndex: Number(env.TV_TEST_SHARD_INDEX ?? "1"), + attemptId: env.TV_TEST_SHARD_ATTEMPT ?? "unavailable", + surfaceId, + } }); +} + +export function startResourceDiagnostics({ root, identity, readProbe = readLinuxResourceProbe, now = () => new Date(), write, warn = warnDefault }) { + const name = `surface-${randomUUID()}`; + const tmpdir = os.tmpdir(); + const start = observe(readProbe, now, tmpdir, warn); + const record = { + kind: "surface", identity: cleanIdentity(identity), start, + end: { status: "missing", reason: "parent-did-not-finalize" }, + nativeEvidence: { status: "pending", failedAttempt: null, recovered: null }, + oomDelta: { value: null, unit: "count", reason: "missing-final-observation" }, + kernel: { status: "not-requested", reason: "parent-did-not-finalize" }, + }; + let file = writeResourceDiagnostic(root, name, record, { write, warn }); + return { + get file() { return file; }, + finish(outcome) { + try { + record.end = observe(readProbe, now, tmpdir, warn); + record.oomDelta = counterDelta(start.oomKills, record.end.oomKills); + record.nativeEvidence = nativeEvidence(outcome, path.join(root, "resources"), warn); + if (outcome.exitCode !== 0 || outcome.incomplete || record.nativeEvidence.status !== "complete" || record.nativeEvidence.failedAttempt) { + const interval = { since: start.observedAt, until: record.end.observedAt }; + try { record.kernel = cleanKernel(readProbe("kernelOOM", interval)); } + catch (error) { record.kernel = { status: "unavailable", reason: errorCode(error) }; warn(`kernel evidence unavailable: ${errorCode(error)}`); } + } else record.kernel = { status: "not-requested", reason: "no-failed-native-attempt" }; + file = writeResourceDiagnostic(root, name, record, { write, warn }) ?? file; + } catch (error) { warn(`final observation unavailable: ${errorCode(error)}`); } + return file && fs.existsSync(file) ? [file] : []; + }, + }; +} + +function observe(readProbe, now, tmpdir, warn) { + const observedAt = now().toISOString(); + const sample = { status: "observed", observedAt }; + for (const [name, metadata] of Object.entries(probes)) { + const value = { ...metadata, observedAt, ...(name === "temporaryFree" ? { path: tmpdir } : {}) }; + try { + const reading = readProbe(name, { tmpdir }); + if (!Number.isSafeInteger(reading) || reading < 0) throw new Error("Invalid numeric probe"); + sample[name] = { ...value, value: reading }; + } catch (error) { + sample[name] = { ...value, value: null, reason: errorCode(error) }; + warn(`${name} unavailable: ${errorCode(error)}`); + } + } + return sample; +} + +export function readLinuxResourceProbe(name, options = {}) { + if (process.platform !== "linux") throw Object.assign(new Error("Linux required"), { code: "ENOTSUP" }); + if (name === "temporaryFree") { + const stats = fs.statfsSync(options.tmpdir ?? os.tmpdir()); + return stats.bavail * stats.bsize; + } + if (name === "kernelOOM") { + const result = spawnSync("dmesg", ["--kernel", "--time-format", "iso", "--since", options.since, "--until", options.until], { encoding: "utf8", timeout: 2000, maxBuffer: 128 * 1024 }); + const truncated = result.error?.code === "ENOBUFS"; + if (!truncated && (result.error || result.status !== 0)) throw Object.assign(new Error("Kernel log unavailable"), { code: result.error?.code ?? "EPERM" }); + return { lines: (result.stdout ?? "").split("\n"), truncated }; + } + const file = name === "oomKills" ? "/proc/vmstat" : "/proc/meminfo"; + const key = name === "oomKills" ? "oom_kill" : name === "memoryAvailable" ? "MemAvailable:" : "Shmem:"; + const line = fs.readFileSync(file, "utf8").split("\n").find((line) => line.startsWith(`${key} `)); + if (!line) throw Object.assign(new Error("Counter unavailable"), { code: "ENOENT" }); + return Number(line.trim().split(/\s+/)[1]) * (name === "oomKills" ? 1 : 1024); +} + +function counterDelta(start, end) { + if (start.value === null || end.value === null) return { value: null, unit: "count", reason: "unavailable" }; + if (end.value < start.value) return { value: null, unit: "count", reason: "counter-reset" }; + return { value: end.value - start.value, unit: "count" }; +} + +function nativeEvidence(outcome, diagnosticDir, warn) { + const nativeExists = outcome.nativeResultPath && fs.existsSync(outcome.nativeResultPath); + const attemptExists = outcome.attemptResultPath && fs.existsSync(outcome.attemptResultPath); + const references = { + resultPath: nativeExists ? path.relative(diagnosticDir, outcome.nativeResultPath) : null, + attemptResultPath: attemptExists ? path.relative(diagnosticDir, outcome.attemptResultPath) : null, + }; + if (!nativeExists) return { ...references, status: "missing", failedAttempt: null, recovered: null }; + try { + const native = normalizeSurfaceResult({ surface: { id: "diagnostic", runner: outcome.runner }, command: [], logPath: null, durationMs: 0, ...outcome }); + const attempts = native.files.flatMap((file) => file.attempts ?? []); + const complete = native.infraStatus !== "incomplete" && native.files.every((file) => file.complete); + return { ...references, status: complete ? "complete" : "incomplete", failedAttempt: attempts.some((attempt) => ["failed", "timed-out", "interrupted"].includes(attempt.status)) || native.counts.testsFailed > 0, recovered: native.counts.testsFlakyRecovered > 0 }; + } catch (error) { + warn(`native attempt evidence unavailable: ${errorCode(error)}`); + return { ...references, status: "incomplete", failedAttempt: null, recovered: null }; + } +} + +// Kernel output is not a trusted text channel. Retain only timestamps, executable +// names and numeric OOM fields, never raw messages, paths, arguments or values. +function cleanKernel(input) { + const lines = []; + let bytes = 0; + let omitted = integer(input?.omittedUnrecognizedLines) ?? 0; + let truncated = Boolean(input?.truncated); + for (const raw of input?.lines ?? []) { + const line = String(raw); + if (!/out of memory|oom[-_ ]kill|killed process/i.test(line)) continue; + const killed = /Killed process (\d+) \(([A-Za-z0-9_.+-]{1,80})\)/.exec(line); + if (!killed) { omitted++; continue; } + const timestamp = /^\d{4}-\d\d-\d\dT[\d:.,+-]+/.exec(line)?.[0]; + const fields = [...line.matchAll(/\b(total-vm|anon-rss|file-rss|shmem-rss|pgtables|UID|oom_score_adj):\s*(-?\d+)(kB)?/g)].map((match) => `${match[1]}:${match[2]}${match[3] ?? ""}`); + const retained = [timestamp, `Killed process ${killed[1]} (${killed[2]})`, ...fields].filter(Boolean).join(" "); + const cost = Buffer.byteLength(retained) + (lines.length ? 1 : 0); + if (bytes + cost > KERNEL_BYTES) { truncated = true; break; } + lines.push(retained); bytes += cost; + } + return { status: "available", lines, truncated, omittedUnrecognizedLines: omitted }; +} + +/** One budget for surface records and recovery receipts in the current attempt. */ +export function writeResourceDiagnostic(root, name, input, { write = atomicWrite, warn = warnDefault } = {}) { + try { + if (!/^[A-Za-z0-9_.-]+$/.test(name)) throw new Error("Invalid diagnostic filename"); + const directory = path.resolve(root, "resources"); + fs.mkdirSync(directory, { recursive: true }); + const file = path.join(directory, `${name}.json`); + // Reserve each baseline's final compact observation before optional excerpts + // can consume the remaining budget. Parents write sequentially per attempt. + const otherUsage = fs.readdirSync(directory).filter((entry) => entry.endsWith(".json") && path.join(directory, entry) !== file) + .reduce((sum, entry) => sum + Math.max(COMPACT_RESERVE_BYTES, fs.statSync(path.join(directory, entry)).size), 0); + const budget = Math.min(RESOURCE_SIDECAR_BYTES, RESOURCE_ATTEMPT_BYTES - otherUsage); + if (budget < COMPACT_RESERVE_BYTES) { warn("attempt budget exhausted; diagnostic omitted"); return null; } + const record = cleanRecord(input); + let contents = serialize(record); + if (Buffer.byteLength(contents) > budget) { + record.truncated = true; + const field = record.kind === "recovery" ? "processes" : null; + const values = field ? record[field] : record.kernel?.lines ?? []; + if (!field && record.kernel) record.kernel.truncated = true; + let low = 0; let high = values.length; + while (low < high) { + const count = Math.ceil((low + high) / 2); + if (field) record[field] = values.slice(0, count); else if (record.kernel) record.kernel.lines = values.slice(0, count); + if (Buffer.byteLength(serialize(record)) <= budget) low = count; else high = count - 1; + } + if (field) { record[field] = values.slice(0, low); record.omittedProcesses = values.length - low; } + else if (record.kernel) record.kernel.lines = values.slice(0, low); + contents = serialize(record); + // The omission count itself can add a few bytes at the boundary. + while (Buffer.byteLength(contents) > budget && field && record[field].length) { record[field].pop(); record.omittedProcesses++; contents = serialize(record); } + warn("diagnostic truncated to its byte budget"); + } + if (Buffer.byteLength(contents) > budget) { warn("compact diagnostic exceeds its budget; omitted"); return null; } + write(file, contents); + return fs.existsSync(file) ? file : null; + } catch (error) { warn(`diagnostic write failed: ${errorCode(error)}`); return null; } +} + +function cleanRecord(input) { + const base = { schemaVersion: 1, kind: input.kind === "recovery" ? "recovery" : "surface", identity: cleanIdentity(input.identity ?? {}) }; + if (base.kind === "recovery") return { + ...base, + previous: select(input.previous, ["runId", "lockId", "worker", "lastRenewal", "recoverAfter"]), + cleanup: token(input.cleanup), discardedReports: Boolean(input.discardedReports), + processes: (input.processes ?? []).map((entry) => ({ pid: integer(entry.pid), startTime: token(entry.startTime), command: executable(entry.command), outcome: token(entry.outcome) })), + }; + return { + ...base, start: cleanObservation(input.start), end: cleanObservation(input.end), + oomDelta: select(input.oomDelta, ["value", "unit", "reason"]), + nativeEvidence: select(input.nativeEvidence, ["status", "failedAttempt", "recovered", "resultPath", "attemptResultPath"]), + kernel: input.kernel?.status === "available" ? cleanKernel(input.kernel) : select(input.kernel, ["status", "reason"]), + }; +} + +function cleanObservation(input) { + const record = select(input, ["status", "reason", "observedAt"]); + for (const key of Object.keys(probes)) if (input?.[key]) record[key] = select(input[key], ["value", "unit", "source", "observedAt", "path", "reason"]); + return record; } -export function writeResourceDiagnostic(root, name, record) { - const file = path.join(root, "resources", `${name}.json`); - fs.mkdirSync(path.dirname(file), { recursive: true }); - fs.writeFileSync(file, JSON.stringify(record)); - return file; +function cleanIdentity(input) { return { runId: token(input.runId), shardIndex: integer(input.shardIndex), attemptId: token(input.attemptId), surfaceId: token(input.surfaceId) }; } +function token(value) { return typeof value === "string" ? value.replace(/[^a-zA-Z0-9_.:/@+-]/g, "_").slice(0, 256) : null; } +function executable(value) { return typeof value === "string" ? token(path.basename(value).split(/\s/)[0]) : null; } +function integer(value) { return Number.isSafeInteger(value) && value >= 0 ? value : null; } +function select(input, fields) { return Object.fromEntries(fields.filter((key) => input?.[key] !== undefined).map((key) => [key, input[key]])); } +function serialize(value) { return `${JSON.stringify(value)}\n`; } +function errorCode(error) { return ["EACCES", "EPERM", "ENOENT", "ENOSPC", "EROFS", "EIO", "ENOTSUP", "ETIMEDOUT", "ENOBUFS"].includes(error?.code) ? error.code : "unavailable"; } +function atomicWrite(file, contents) { + const temporary = `${file}.${process.pid}.tmp`; + try { fs.writeFileSync(temporary, contents); fs.renameSync(temporary, file); } + finally { fs.rmSync(temporary, { force: true }); } } diff --git a/scripts/test/supervised-command.mjs b/scripts/test/supervised-command.mjs index f2b9fa7e..761a25a7 100644 --- a/scripts/test/supervised-command.mjs +++ b/scripts/test/supervised-command.mjs @@ -4,17 +4,29 @@ import path from "node:path"; import process from "node:process"; import { createSurfaceSupervisor } from "./surface-supervisor.mjs"; import { startSurfaceServices } from "./surface-services.mjs"; +import { writeVitestRunnerConfig } from "./execution-groups.mjs"; +import { surfaceResourceDiagnostics } from "./resource-diagnostics.mjs"; const { runId, surfaceIds, resultPath, preCommand, services, command, args } = parseArgs(process.argv.slice(2)); const supervisor = createSurfaceSupervisor({ runId, surfaceIds }); +const diagnostics = surfaceResourceDiagnostics({ resultsDir: path.dirname(resultPath), surfaceId: surfaceIds.join(","), runId }); +let executionEnv = process.env; +let attemptResultPath = null; let commandResult = { exitCode: 127, signal: null, error: null }; let cleanupError = null; let executionFailure = null; let processLeaks = []; let serviceScope = null; try { + if (process.env.TV_TARGET_RUNNER === "vitest" && process.env.TV_TARGET_CONFIG) { + const runnerConfig = path.join(path.dirname(resultPath), "target.runner.config.mjs"); + attemptResultPath = `${process.env.TV_TARGET_NATIVE_RESULT}.attempts.ndjson`; + fs.rmSync(attemptResultPath, { force: true }); + writeVitestRunnerConfig({ configPath: runnerConfig, baseConfigPath: process.env.TV_TARGET_CONFIG, runnerPath: "scripts/test/vitest-attempt-reporter.mjs" }); + executionEnv = { ...process.env, TV_TARGET_VITEST_CONFIG: runnerConfig, TV_VITEST_ATTEMPT_FILE: attemptResultPath }; + } if (preCommand) { - const preResult = await supervisor.run(preCommand[0], preCommand.slice(1), { cwd: process.cwd(), env: process.env, stdio: "inherit" }); + const preResult = await supervisor.run(preCommand[0], preCommand.slice(1), { cwd: process.cwd(), env: executionEnv, stdio: "inherit" }); const preLeaks = await supervisor.checkForLeaks(); if (preResult.exitCode !== 0 || preLeaks.length > 0) { commandResult = preResult; @@ -23,14 +35,14 @@ try { } if (!executionFailure && services.length > 0) { try { - serviceScope = await startSurfaceServices(services, { env: supervisor.childEnv(process.env) }); + serviceScope = await startSurfaceServices(services, { env: supervisor.childEnv(executionEnv) }); } catch (error) { commandResult = { exitCode: 1, signal: null, error }; executionFailure = { kind: "surface-service", message: error.message ?? String(error), step: "surface-service-start" }; } } if (!executionFailure) { - commandResult = await supervisor.run(command, args, { cwd: process.cwd(), env: serviceScope?.env ?? process.env, stdio: "inherit" }); + commandResult = await supervisor.run(command, args, { cwd: process.cwd(), env: serviceScope?.env ?? executionEnv, stdio: "inherit" }); } } finally { if (serviceScope) { @@ -50,6 +62,7 @@ try { } const failedForLifecycle = processLeaks.length > 0 || cleanupError !== null; +const resourceDiagnosticsPaths = diagnostics.finish({ runner: process.env.TV_TARGET_RUNNER, nativeResultPath: process.env.TV_TARGET_NATIVE_RESULT, attemptResultPath, exitCode: commandResult.exitCode, incomplete: failedForLifecycle || Boolean(executionFailure) }); const publishedServices = serviceScope?.services ?? []; const result = { schemaVersion: 1, @@ -62,6 +75,8 @@ const result = { exitCode: commandResult.exitCode, signal: commandResult.signal, processLeaks, + resourceDiagnosticsPaths, + attemptResultPath: attemptResultPath && fs.existsSync(attemptResultPath) ? attemptResultPath : null, failureKind: cleanupError?.name === "UnconfirmedSurfaceCleanupError" ? "unconfirmed-process-cleanup" : executionFailure?.kind ?? (failedForLifecycle ? "process-leak" : null), failureStep: executionFailure?.step ?? null, failureMessage: cleanupError?.message ?? executionFailure?.message ?? (processLeaks.length > 0 ? `${processLeaks.length} owned process leak(s) were reaped` : null), From 86bdf6da879087051e7873177738181da2e2ecb1 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:01:59 +0000 Subject: [PATCH 23/64] Exercise abrupt parent death and reserved final observations --- test/fixtures/resource-diagnostics/prepare.mjs | 6 +++++- test/repo/resource-diagnostics-links.test.ts | 2 +- test/repo/resource-diagnostics.test.ts | 3 +++ test/repo/test-runner-resource-parent.test.ts | 1 + 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/test/fixtures/resource-diagnostics/prepare.mjs b/test/fixtures/resource-diagnostics/prepare.mjs index 57037663..b1f8b222 100644 --- a/test/fixtures/resource-diagnostics/prepare.mjs +++ b/test/fixtures/resource-diagnostics/prepare.mjs @@ -5,4 +5,8 @@ const root = process.env.RESOURCE_FIXTURE_ROOT; const resources = path.join(root, "results", "resources"); const records = fs.existsSync(resources) ? fs.readdirSync(resources).filter((name) => name.endsWith(".json")).map((name) => JSON.parse(fs.readFileSync(path.join(resources, name), "utf8"))) : []; fs.writeFileSync(path.join(root, "preparation.json"), JSON.stringify({ observedAt: new Date().toISOString(), records })); -if (process.env.RESOURCE_FIXTURE_MODE === "interrupt") await new Promise((resolve) => setTimeout(resolve, 30_000)); +if (process.env.RESOURCE_FIXTURE_MODE === "interrupt") { + // Abruptly end our own fixture parent, then exit this only descendant too. + // TERM permits the real supervisor to finalize, which is a different path. + process.kill(process.ppid, "SIGKILL"); +} diff --git a/test/repo/resource-diagnostics-links.test.ts b/test/repo/resource-diagnostics-links.test.ts index 67a90d22..7938f384 100644 --- a/test/repo/resource-diagnostics-links.test.ts +++ b/test/repo/resource-diagnostics-links.test.ts @@ -33,7 +33,7 @@ test("keeps only retained resource links across infrastructure replacement, norm expect(results.surfaces[0].resourceDiagnosticsPaths).toEqual(["attempt-0/resources.json", "attempt-1/resources.json"]); for (const link of results.surfaces[0].resourceDiagnosticsPaths as string[]) expect(existsSync(path.join(root, link))).toBe(true); expect(summary.run.status).toBe("passed"); - const events = serializeTimingEvents(buildTimingEvents({ run: { id: "resource-run", commandProvider: "blaxel", timingProvider: "blaxel-playwright-x64-4vcpu", status: "passed", startedAt: start, completedAt: end, durationMs: 1000, selection: { surfaces: ["e2e:fixture"] }, git: { commit: null, tree: null, dirty: false }, environment: { ci: true, os: "linux", arch: "x64", node: process.version }, phases: [] }, shards: [{ index: 1, total: 1, status: "passed", phases: [], surfaces: normalized as any }] })); + const events = serializeTimingEvents(buildTimingEvents({ run: { runId: "resource-run", commandProvider: "blaxel", timingProvider: "blaxel-playwright-x64-4vcpu", status: "passed", startedAt: start, completedAt: end, durationMs: 1000, selection: { surfaces: ["e2e:fixture"] }, git: { commit: null, tree: null, dirty: false }, environment: { ci: true, os: "linux", arch: "x64", node: process.version }, phases: [] }, shards: [{ index: 1, total: 1, status: "passed", phases: [], surfaces: normalized as any }] })); for (const compact of [JSON.stringify(summary), JSON.stringify(compactProviderSummary(combined)), events]) { expect(compact).not.toContain("KERNEL-DETAIL-SENTINEL"); expect(compact).not.toContain("memoryAvailable"); diff --git a/test/repo/resource-diagnostics.test.ts b/test/repo/resource-diagnostics.test.ts index 7e5426eb..f3671ff2 100644 --- a/test/repo/resource-diagnostics.test.ts +++ b/test/repo/resource-diagnostics.test.ts @@ -64,8 +64,11 @@ test("bounds sidecars and the attempt including receipts, reserving compact obse expect(record.kernel.truncated).toBe(true); expect(Buffer.byteLength(record.kernel.lines.join("\n"))).toBeLessThanOrEqual(64 * 1024); expect(statSync(collector.file!).size).toBeLessThanOrEqual(128 * 1024); + const pending = startResourceDiagnostics({ root, identity, readProbe: () => 2, warn: (message) => warnings.push(message) }); const receipt = { kind: "recovery", identity, previous: { runId: "old", lockId: "old-lock", worker: "private-worker", lastRenewal: startTime, recoverAfter: endTime, args: secret }, discardedReports: true, cleanup: "confirmed", processes: Array.from({ length: 3000 }, (_, i) => ({ pid: i + 1, startTime: "123", command: "node", outcome: "terminated", args: secret })), env: secret }; for (let i = 0; i < 80; i++) writeResourceDiagnostic(root, `receipt-${i}`, receipt, { warn: (message) => warnings.push(message) }); + pending.finish({ runner: "vitest", nativeResultPath: null, exitCode: 17 }); + expect(read(pending.file).end.memoryAvailable.value).toBe(2); const files = readdirSync(path.join(root, "resources")).map((name) => path.join(root, "resources", name)); expect(files.reduce((sum, file) => sum + statSync(file).size, 0)).toBeLessThanOrEqual(8 * 1024 * 1024); expect(files.every((file) => statSync(file).size <= 128 * 1024)).toBe(true); diff --git a/test/repo/test-runner-resource-parent.test.ts b/test/repo/test-runner-resource-parent.test.ts index 1cf2bebe..83c83214 100644 --- a/test/repo/test-runner-resource-parent.test.ts +++ b/test/repo/test-runner-resource-parent.test.ts @@ -64,6 +64,7 @@ test("interrupted surface parent leaves an explicit missing final observation", const child = spawnOwnedProcess(process.execPath, fixture.args, { env: fixture.env, stdio: "ignore" }); try { await expect.poll(() => existsSync(path.join(fixture.dir, "preparation.json")), { timeout: 10_000, interval: 20 }).toBe(true); + await expect.poll(() => child.child.signalCode, { timeout: 5_000, interval: 20 }).toBe("SIGKILL"); expect((await child.dispose()).outcome).not.toBe("survived"); const records = readDiagnostics(fixture.dir); expect(records).toHaveLength(1); From 83e8c9f13471f6b2d767ca89b3b5cc2a7f9a337c Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:05:11 +0000 Subject: [PATCH 24/64] Avoid inferring permission failure from any kernel-reader exit --- scripts/test/resource-diagnostics.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/test/resource-diagnostics.mjs b/scripts/test/resource-diagnostics.mjs index 2d50538d..18586195 100644 --- a/scripts/test/resource-diagnostics.mjs +++ b/scripts/test/resource-diagnostics.mjs @@ -85,7 +85,10 @@ export function readLinuxResourceProbe(name, options = {}) { if (name === "kernelOOM") { const result = spawnSync("dmesg", ["--kernel", "--time-format", "iso", "--since", options.since, "--until", options.until], { encoding: "utf8", timeout: 2000, maxBuffer: 128 * 1024 }); const truncated = result.error?.code === "ENOBUFS"; - if (!truncated && (result.error || result.status !== 0)) throw Object.assign(new Error("Kernel log unavailable"), { code: result.error?.code ?? "EPERM" }); + if (!truncated && (result.error || result.status !== 0)) { + const permissionCode = /operation not permitted/i.test(result.stderr ?? "") ? "EPERM" : /permission denied/i.test(result.stderr ?? "") ? "EACCES" : "unavailable"; + throw Object.assign(new Error("Kernel log unavailable"), { code: result.error?.code ?? permissionCode }); + } return { lines: (result.stdout ?? "").split("\n"), truncated }; } const file = name === "oomKills" ? "/proc/vmstat" : "/proc/meminfo"; From f72654e17511e6245b17a432b69ff38440358dd8 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:06:24 +0000 Subject: [PATCH 25/64] Distinguish diagnostic identities across infrastructure attempts --- scripts/run-blaxel-testshards.mjs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index c77df60c..71fc7aaa 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -267,6 +267,7 @@ async function runAttempt({ attempt, shardIndices, excludedSandboxes = [], outpu console.log(`starting: ${formatShardLeases(acquired)}`); results = await Promise.all(acquired.map(async (lease) => { const result = await runShard(lease, outputDir, { + attempt, onExecutionRequested: () => attemptState.executionRequested.add(lease.shardIndex), onExecutionStarted: () => attemptState.executionStarted.add(lease.shardIndex), onResultProgress: (progress) => attemptState.partialResults.set(lease.shardIndex, progress), @@ -410,8 +411,9 @@ function lockCommand({ lockId, shardIndex, expiresAt }) { ].join("; "); } -async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onExecutionStarted = () => {}, onResultProgress = () => {} } = {}) { +async function runShard(lease, outputDir, { attempt, onExecutionRequested = () => {}, onExecutionStarted = () => {}, onResultProgress = () => {} } = {}) { const { sandbox, shardIndex, lockId } = lease; + const diagnosticAttemptId = `${lockId}-attempt-${attempt + 1}`; const startedAt = Date.now(); if (lease.restorePhase) { lease.restorePhase.completedAt = new Date(startedAt).toISOString(); @@ -430,7 +432,7 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE planRemotePath, planRaw: shardPlanRaw, remoteScript, - remoteScriptContents: shardScript({ shardIndex, lockId, planRemotePath }), + remoteScriptContents: shardScript({ shardIndex, lockId, planRemotePath, diagnosticAttemptId }), }); if (!dispatch.ok) { appendLog(logPath, `\n[coordinator] ${dispatch.failureMessage}\n`); @@ -529,7 +531,7 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE } } catch {} retainResultProgress(); - resourceDiagnosticsPaths = await copyResourceDiagnostics(sandbox, { outputDir, shardIndex, lockId, copiedArtifacts }); + resourceDiagnosticsPaths = await copyResourceDiagnostics(sandbox, { outputDir, shardIndex, diagnosticAttemptId, copiedArtifacts }); retainResultProgress(); for (const [taskIndex, task] of (shardSummary?.tasks ?? []).entries()) { const copiedTask = { ...tasks[taskIndex] }; @@ -578,7 +580,7 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE return { shardIndex, shardTotal, shard: `${shardIndex}/${shardTotal}`, sandbox: sandbox.metadata.name, outputDir, status, exitCode, startedAt: new Date(overallStarted).toISOString(), completedAt, durationMs: Date.now() - overallStarted, summaryPath: fs.existsSync(summaryPath) ? summaryPath : null, summaryError: shardSummaryError, nativeResultPath, resourceDiagnosticsPaths, logPath, setupTimings, planId: shardSummary?.planId ?? shardPlan?.planId, timingProvider: shardSummary?.timingProvider ?? shardPlan?.timingProvider, testedTree: shardSummary?.testedTree ?? shardPlan?.testedTree, testedCommit: shardSummary?.testedCommit ?? shardPlan?.testedCommit, predictedDurationMs: shardSummary?.predictedDurationMs, phases: [...(shardSummary?.phases ?? []), reportPhase], assignedFiles: shardSummary?.assignedFiles ?? [], collectedFiles: shardSummary?.collectedFiles ?? [], failureKind: identityError ? "plan-validation" : resultError ? "result-report" : missingSummaryFailure?.failureKind ?? shardSummary?.failureKind ?? failedInfrastructureTask?.failureKind, failureStep: identityError ? "plan-validation" : resultError ? "summary-validation" : missingSummaryFailure?.failureStep ?? shardSummary?.failureStep ?? failedInfrastructureTask?.failureStep, failureMessage: identityError ?? resultError ?? missingSummaryFailure?.failureMessage ?? shardSummary?.failureMessage ?? failedInfrastructureTask?.failureMessage, processLeaks: shardSummary?.processLeaks ?? [], tasks }; } catch (error) { stream?.close?.(); - resourceDiagnosticsPaths = await copyResourceDiagnostics(sandbox, { outputDir, shardIndex, lockId, copiedArtifacts }); + resourceDiagnosticsPaths = await copyResourceDiagnostics(sandbox, { outputDir, shardIndex, diagnosticAttemptId, copiedArtifacts }); appendLog(logPath, `\n[coordinator] ${error.stack || error.message}\n`); const completedAt = new Date().toISOString(); const plannedShard = shardPlan?.shards?.[shardIndex - 1]; @@ -659,7 +661,7 @@ function retainTaskWithoutRemoteArtifacts(task) { return retained; } -async function copyResourceDiagnostics(sandbox, { outputDir, shardIndex, lockId, copiedArtifacts }) { +async function copyResourceDiagnostics(sandbox, { outputDir, shardIndex, diagnosticAttemptId, copiedArtifacts }) { const root = "/workspace/television/.testshards/results/resources"; const retained = []; let bytes = 0; @@ -674,7 +676,7 @@ async function copyResourceDiagnostics(sandbox, { outputDir, shardIndex, lockId, const size = Buffer.byteLength(raw); if (size > RESOURCE_SIDECAR_BYTES || bytes + size > RESOURCE_ATTEMPT_BYTES) throw new Error("budget"); const record = JSON.parse(raw); - if (record.schemaVersion !== 1 || record.identity?.runId !== runId || record.identity?.shardIndex !== shardIndex || record.identity?.attemptId !== lockId) throw new Error("identity"); + if (record.schemaVersion !== 1 || record.identity?.runId !== runId || record.identity?.shardIndex !== shardIndex || record.identity?.attemptId !== diagnosticAttemptId) throw new Error("identity"); const localPath = path.join(outputDir, "artifacts", `shard-${shardIndex}`, "resources", entry.name); fs.mkdirSync(path.dirname(localPath), { recursive: true }); fs.writeFileSync(localPath, raw); @@ -805,7 +807,7 @@ function systemDependenciesCommand() { return 'if ! command -v python3 >/dev/null 2>&1 || ! command -v xauth >/dev/null 2>&1 || ! command -v ss >/dev/null 2>&1 || ! ldconfig -p 2>/dev/null | grep -q "libgtk-3.so.0" || ! ldconfig -p 2>/dev/null | grep -q "libXtst.so.6"; then apt-get update && apt-get install -y --no-install-recommends python3 xauth libgtk-3-0 libxtst6 iproute2; fi'; } -function shardScript({ shardIndex, lockId, planRemotePath }) { +function shardScript({ shardIndex, lockId, planRemotePath, diagnosticAttemptId }) { return `#!/usr/bin/env bash set -euo pipefail lock=/tmp/tv-testshard.lock @@ -824,7 +826,7 @@ export TEST_RETRIES=${testRetries} export TV_TEST_TIMING_PROVIDER=blaxel-playwright-x64-4vcpu export TV_TEST_RUN_ID=${shellQuote(runId)} export TV_TEST_SHARD_INDEX=${shardIndex} -export TV_TEST_SHARD_ATTEMPT=${shellQuote(lockId)} +export TV_TEST_SHARD_ATTEMPT=${shellQuote(diagnosticAttemptId)} mkdir -p /workspace /cache/npm if [ -z "\${GH_TOKEN:-}" ]; then echo "GH_TOKEN missing" >&2; exit 2; fi setup_metrics=/tmp/tv-testshard-setup-${shardIndex}.jsonl From 0db76dd547d91d4ef58746a433e9d35fb87683cc Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:10:40 +0000 Subject: [PATCH 26/64] Require an attributable kernel-reader limitation in the real seam --- test/repo/test-runner-resource-parent.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/repo/test-runner-resource-parent.test.ts b/test/repo/test-runner-resource-parent.test.ts index 83c83214..07e6f719 100644 --- a/test/repo/test-runner-resource-parent.test.ts +++ b/test/repo/test-runner-resource-parent.test.ts @@ -35,7 +35,11 @@ for (const entry of ["planned", "targeted"] as const) { } expect(record.nativeEvidence).toMatchObject({ status: "complete", failedAttempt: true, recovered: true }); expect(["available", "unavailable"]).toContain(record.kernel.status); - if (record.kernel.status === "unavailable") expect(record.kernel.reason).toEqual(expect.any(String)); + if (record.kernel.status === "unavailable") { + // Expected host limitations stay unknown; malformed reader invocations + // must not masquerade as a successful probe of this installed tool. + expect(["EPERM", "EACCES", "ENOENT", "ENOTSUP", "ETIMEDOUT", "ENOBUFS"]).toContain(record.kernel.reason); + } const resultRecord = entry === "planned" ? read(path.join(fixture.dir, "results/shard-1.json")).tasks[0] : read(path.join(fixture.dir, "results/lifecycle.json")); expect(resultRecord.resourceDiagnosticsPaths).toHaveLength(1); expect(existsSync(resultRecord.resourceDiagnosticsPaths[0])).toBe(true); From cf4fb510917699b4402e34c013457c158db1b0fe Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:11:26 +0000 Subject: [PATCH 27/64] Use epoch bounds supported by the installed kernel log reader --- scripts/test/resource-diagnostics.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/test/resource-diagnostics.mjs b/scripts/test/resource-diagnostics.mjs index 18586195..99ebeb49 100644 --- a/scripts/test/resource-diagnostics.mjs +++ b/scripts/test/resource-diagnostics.mjs @@ -83,7 +83,10 @@ export function readLinuxResourceProbe(name, options = {}) { return stats.bavail * stats.bsize; } if (name === "kernelOOM") { - const result = spawnSync("dmesg", ["--kernel", "--time-format", "iso", "--since", options.since, "--until", options.until], { encoding: "utf8", timeout: 2000, maxBuffer: 128 * 1024 }); + // util-linux accepts fractional epoch seconds, not ISO's trailing Z. + // Epoch bounds also avoid interpreting UTC observations in local time. + const epoch = (value) => `@${(Date.parse(value) / 1000).toFixed(3)}`; + const result = spawnSync("dmesg", ["--kernel", "--time-format", "iso", "--since", epoch(options.since), "--until", epoch(options.until)], { encoding: "utf8", timeout: 2000, maxBuffer: 128 * 1024 }); const truncated = result.error?.code === "ENOBUFS"; if (!truncated && (result.error || result.status !== 0)) { const permissionCode = /operation not permitted/i.test(result.stderr ?? "") ? "EPERM" : /permission denied/i.test(result.stderr ?? "") ? "EACCES" : "unavailable"; From 734a29ac9e344c5e714a85c61fcfaef248f2aaff Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:13:53 +0000 Subject: [PATCH 28/64] Expose the unexpected kernel-reader error in the focused seam --- test/repo/test-runner-resource-parent.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/repo/test-runner-resource-parent.test.ts b/test/repo/test-runner-resource-parent.test.ts index 07e6f719..52b15995 100644 --- a/test/repo/test-runner-resource-parent.test.ts +++ b/test/repo/test-runner-resource-parent.test.ts @@ -38,7 +38,8 @@ for (const entry of ["planned", "targeted"] as const) { if (record.kernel.status === "unavailable") { // Expected host limitations stay unknown; malformed reader invocations // must not masquerade as a successful probe of this installed tool. - expect(["EPERM", "EACCES", "ENOENT", "ENOTSUP", "ETIMEDOUT", "ENOBUFS"]).toContain(record.kernel.reason); + const debug = record.kernel.reason === "unavailable" ? spawnSync("dmesg", ["--kernel", "--time-format", "iso", "--since", `@${Date.parse(record.start.observedAt) / 1000}`, "--until", `@${Date.parse(record.end.observedAt) / 1000}`], { encoding: "utf8", timeout: 2000, maxBuffer: 128 * 1024 }).stderr : ""; + expect(["EPERM", "EACCES", "ENOENT", "ENOTSUP", "ETIMEDOUT", "ENOBUFS"], debug).toContain(record.kernel.reason); } const resultRecord = entry === "planned" ? read(path.join(fixture.dir, "results/shard-1.json")).tasks[0] : read(path.join(fixture.dir, "results/lifecycle.json")); expect(resultRecord.resourceDiagnosticsPaths).toHaveLength(1); From dcc0847c25a457de2df1d099e0d23192b7f503ae Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:15:18 +0000 Subject: [PATCH 29/64] Use UTC calendar bounds accepted by the worker kernel reader --- scripts/test/resource-diagnostics.mjs | 8 ++++---- test/repo/test-runner-resource-parent.test.ts | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/test/resource-diagnostics.mjs b/scripts/test/resource-diagnostics.mjs index 99ebeb49..02fd8502 100644 --- a/scripts/test/resource-diagnostics.mjs +++ b/scripts/test/resource-diagnostics.mjs @@ -83,10 +83,10 @@ export function readLinuxResourceProbe(name, options = {}) { return stats.bavail * stats.bsize; } if (name === "kernelOOM") { - // util-linux accepts fractional epoch seconds, not ISO's trailing Z. - // Epoch bounds also avoid interpreting UTC observations in local time. - const epoch = (value) => `@${(Date.parse(value) / 1000).toFixed(3)}`; - const result = spawnSync("dmesg", ["--kernel", "--time-format", "iso", "--since", epoch(options.since), "--until", epoch(options.until)], { encoding: "utf8", timeout: 2000, maxBuffer: 128 * 1024 }); + // Calendar bounds work with the worker's util-linux/libc time parser; + // explicit UTC prevents local-time interpretation of our ISO observations. + const calendar = (value) => new Date(value).toISOString().slice(0, -1).replace("T", " "); + const result = spawnSync("dmesg", ["--kernel", "--time-format", "iso", "--since", calendar(options.since), "--until", calendar(options.until)], { encoding: "utf8", env: { ...process.env, TZ: "UTC", LC_ALL: "C" }, timeout: 2000, maxBuffer: 128 * 1024 }); const truncated = result.error?.code === "ENOBUFS"; if (!truncated && (result.error || result.status !== 0)) { const permissionCode = /operation not permitted/i.test(result.stderr ?? "") ? "EPERM" : /permission denied/i.test(result.stderr ?? "") ? "EACCES" : "unavailable"; diff --git a/test/repo/test-runner-resource-parent.test.ts b/test/repo/test-runner-resource-parent.test.ts index 52b15995..07e6f719 100644 --- a/test/repo/test-runner-resource-parent.test.ts +++ b/test/repo/test-runner-resource-parent.test.ts @@ -38,8 +38,7 @@ for (const entry of ["planned", "targeted"] as const) { if (record.kernel.status === "unavailable") { // Expected host limitations stay unknown; malformed reader invocations // must not masquerade as a successful probe of this installed tool. - const debug = record.kernel.reason === "unavailable" ? spawnSync("dmesg", ["--kernel", "--time-format", "iso", "--since", `@${Date.parse(record.start.observedAt) / 1000}`, "--until", `@${Date.parse(record.end.observedAt) / 1000}`], { encoding: "utf8", timeout: 2000, maxBuffer: 128 * 1024 }).stderr : ""; - expect(["EPERM", "EACCES", "ENOENT", "ENOTSUP", "ETIMEDOUT", "ENOBUFS"], debug).toContain(record.kernel.reason); + expect(["EPERM", "EACCES", "ENOENT", "ENOTSUP", "ETIMEDOUT", "ENOBUFS"]).toContain(record.kernel.reason); } const resultRecord = entry === "planned" ? read(path.join(fixture.dir, "results/shard-1.json")).tasks[0] : read(path.join(fixture.dir, "results/lifecycle.json")); expect(resultRecord.resourceDiagnosticsPaths).toHaveLength(1); From 056f9e21bf53eb41b01bfd434098c3a68a1c3be3 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:17:13 +0000 Subject: [PATCH 30/64] Inspect the installed dmesg interval parser on the private worker --- test/repo/test-runner-resource-parent.test.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/repo/test-runner-resource-parent.test.ts b/test/repo/test-runner-resource-parent.test.ts index 07e6f719..c94d0372 100644 --- a/test/repo/test-runner-resource-parent.test.ts +++ b/test/repo/test-runner-resource-parent.test.ts @@ -38,7 +38,16 @@ for (const entry of ["planned", "targeted"] as const) { if (record.kernel.status === "unavailable") { // Expected host limitations stay unknown; malformed reader invocations // must not masquerade as a successful probe of this installed tool. - expect(["EPERM", "EACCES", "ENOENT", "ENOTSUP", "ETIMEDOUT", "ENOBUFS"]).toContain(record.kernel.reason); + const debug = record.kernel.reason === "unavailable" ? [ + ["--version"], + ["--since", record.start.observedAt.slice(0, 19).replace("T", " ")], + ["--since", record.start.observedAt.slice(0, -1).replace("T", " ")], + ["--since", `@${Math.floor(Date.parse(record.start.observedAt) / 1000)}`], + ].map((args) => { + const probe = spawnSync("dmesg", args, { encoding: "utf8", env: { ...process.env, TZ: "UTC", LC_ALL: "C" }, timeout: 2000, maxBuffer: 128 * 1024 }); + return { args, status: probe.status, stderr: probe.stderr, version: args[0] === "--version" ? probe.stdout : undefined }; + }) : []; + expect(["EPERM", "EACCES", "ENOENT", "ENOTSUP", "ETIMEDOUT", "ENOBUFS"], JSON.stringify(debug)).toContain(record.kernel.reason); } const resultRecord = entry === "planned" ? read(path.join(fixture.dir, "results/shard-1.json")).tasks[0] : read(path.join(fixture.dir, "results/lifecycle.json")); expect(resultRecord.resourceDiagnosticsPaths).toHaveLength(1); From b9e9cbf868e754102e29cadaace745e2b95f3230 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:19:33 +0000 Subject: [PATCH 31/64] Require exact interval filtering of coarse kernel reads --- scripts/test/resource-diagnostics.d.mts | 1 + scripts/test/resource-diagnostics.mjs | 4 ++++ test/repo/resource-diagnostics.test.ts | 12 +++++++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/test/resource-diagnostics.d.mts b/scripts/test/resource-diagnostics.d.mts index 783c9390..b69fb4d1 100644 --- a/scripts/test/resource-diagnostics.d.mts +++ b/scripts/test/resource-diagnostics.d.mts @@ -4,3 +4,4 @@ export const RESOURCE_SIDECAR_BYTES: number; export const RESOURCE_ATTEMPT_BYTES: number; export function surfaceResourceDiagnostics(options: { resultsDir: string; surfaceId: string; env?: NodeJS.ProcessEnv; runId: string }): ReturnType; export function readLinuxResourceProbe(name: string, options?: Record): unknown; +export function kernelLinesInInterval(lines: string[], interval: { since: string; until: string }): { lines: string[]; omittedUnrecognizedLines: number }; diff --git a/scripts/test/resource-diagnostics.mjs b/scripts/test/resource-diagnostics.mjs index 02fd8502..02ba51df 100644 --- a/scripts/test/resource-diagnostics.mjs +++ b/scripts/test/resource-diagnostics.mjs @@ -107,6 +107,10 @@ function counterDelta(start, end) { return { value: end.value - start.value, unit: "count" }; } +export function kernelLinesInInterval(lines, interval) { + return { lines, omittedUnrecognizedLines: 0 }; +} + function nativeEvidence(outcome, diagnosticDir, warn) { const nativeExists = outcome.nativeResultPath && fs.existsSync(outcome.nativeResultPath); const attemptExists = outcome.attemptResultPath && fs.existsSync(outcome.attemptResultPath); diff --git a/test/repo/resource-diagnostics.test.ts b/test/repo/resource-diagnostics.test.ts index f3671ff2..43e78a75 100644 --- a/test/repo/resource-diagnostics.test.ts +++ b/test/repo/resource-diagnostics.test.ts @@ -2,7 +2,7 @@ import { copyFileSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSyn import os from "node:os"; import path from "node:path"; import { afterEach, expect, test } from "vitest"; -import { startResourceDiagnostics, writeResourceDiagnostic } from "../../scripts/test/resource-diagnostics.mjs"; +import { kernelLinesInInterval, startResourceDiagnostics, writeResourceDiagnostic } from "../../scripts/test/resource-diagnostics.mjs"; const roots: string[] = []; const identity = { runId: "run-one", shardIndex: 1, attemptId: "lock-one", surfaceId: "e2e:fixture" }; @@ -13,6 +13,16 @@ const temporary = () => { const dir = mkdtempSync(path.join(os.tmpdir(), "tv-res const read = (file: string | null) => JSON.parse(readFileSync(file!, "utf8")); afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +test("restricts a whole-second kernel read to the observed interval and discloses missing timestamps", () => { + const line = (time: string) => `${time} Out of memory: Killed process 23 (node)`; + const within = line("2026-10-27T00:00:00,500000+00:00"); + expect(kernelLinesInInterval([ + line("2026-10-27T00:00:00,100000+00:00"), within, + line("2026-10-27T00:00:00,900000+00:00"), + "Killed process 24 (node)", "unrelated kernel message", + ], { since: "2026-10-27T00:00:00.200Z", until: "2026-10-27T00:00:00.800Z" })).toEqual({ lines: [within], omittedUnrecognizedLines: 1 }); +}); + test("retains a baseline, honest units and reset evidence, and kernel evidence for a recovered native attempt", () => { const root = temporary(); const nativeResultPath = path.join(root, "native.json"); From 0e82df7f003c509c9530ee4d2f47aa015de7be6b Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:20:46 +0000 Subject: [PATCH 32/64] Bound kernel reads by seconds and retain only the observed interval --- scripts/test/resource-diagnostics.mjs | 22 ++++++++++++++----- test/repo/test-runner-resource-parent.test.ts | 11 +--------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/scripts/test/resource-diagnostics.mjs b/scripts/test/resource-diagnostics.mjs index 02ba51df..c83a64fa 100644 --- a/scripts/test/resource-diagnostics.mjs +++ b/scripts/test/resource-diagnostics.mjs @@ -83,16 +83,16 @@ export function readLinuxResourceProbe(name, options = {}) { return stats.bavail * stats.bsize; } if (name === "kernelOOM") { - // Calendar bounds work with the worker's util-linux/libc time parser; - // explicit UTC prevents local-time interpretation of our ISO observations. - const calendar = (value) => new Date(value).toISOString().slice(0, -1).replace("T", " "); - const result = spawnSync("dmesg", ["--kernel", "--time-format", "iso", "--since", calendar(options.since), "--until", calendar(options.until)], { encoding: "utf8", env: { ...process.env, TZ: "UTC", LC_ALL: "C" }, timeout: 2000, maxBuffer: 128 * 1024 }); + // The worker's util-linux 2.38 accepts whole-second calendar bounds. Read + // an enclosing UTC window, then filter its output to our exact interval. + const calendar = (value) => new Date(value).toISOString().slice(0, 19).replace("T", " "); + const result = spawnSync("dmesg", ["--kernel", "--time-format", "iso", "--since", calendar(Date.parse(options.since) - 1), "--until", calendar(Date.parse(options.until) + 1000)], { encoding: "utf8", env: { ...process.env, TZ: "UTC", LC_ALL: "C" }, timeout: 2000, maxBuffer: 128 * 1024 }); const truncated = result.error?.code === "ENOBUFS"; if (!truncated && (result.error || result.status !== 0)) { const permissionCode = /operation not permitted/i.test(result.stderr ?? "") ? "EPERM" : /permission denied/i.test(result.stderr ?? "") ? "EACCES" : "unavailable"; throw Object.assign(new Error("Kernel log unavailable"), { code: result.error?.code ?? permissionCode }); } - return { lines: (result.stdout ?? "").split("\n"), truncated }; + return { ...kernelLinesInInterval((result.stdout ?? "").split("\n"), options), truncated }; } const file = name === "oomKills" ? "/proc/vmstat" : "/proc/meminfo"; const key = name === "oomKills" ? "oom_kill" : name === "memoryAvailable" ? "MemAvailable:" : "Shmem:"; @@ -108,7 +108,17 @@ function counterDelta(start, end) { } export function kernelLinesInInterval(lines, interval) { - return { lines, omittedUnrecognizedLines: 0 }; + const since = Date.parse(interval.since); + const until = Date.parse(interval.until); + let omittedUnrecognizedLines = 0; + const retained = lines.filter((line) => { + if (!/out of memory|oom[-_ ]kill|killed process/i.test(line)) return false; + const timestamp = line.split(/\s/, 1)[0].replace(",", ".").replace(/\.(\d{3})\d+/, ".$1"); + const time = Date.parse(timestamp); + if (!Number.isFinite(time)) { omittedUnrecognizedLines++; return false; } + return time >= since && time <= until; + }); + return { lines: retained, omittedUnrecognizedLines }; } function nativeEvidence(outcome, diagnosticDir, warn) { diff --git a/test/repo/test-runner-resource-parent.test.ts b/test/repo/test-runner-resource-parent.test.ts index c94d0372..07e6f719 100644 --- a/test/repo/test-runner-resource-parent.test.ts +++ b/test/repo/test-runner-resource-parent.test.ts @@ -38,16 +38,7 @@ for (const entry of ["planned", "targeted"] as const) { if (record.kernel.status === "unavailable") { // Expected host limitations stay unknown; malformed reader invocations // must not masquerade as a successful probe of this installed tool. - const debug = record.kernel.reason === "unavailable" ? [ - ["--version"], - ["--since", record.start.observedAt.slice(0, 19).replace("T", " ")], - ["--since", record.start.observedAt.slice(0, -1).replace("T", " ")], - ["--since", `@${Math.floor(Date.parse(record.start.observedAt) / 1000)}`], - ].map((args) => { - const probe = spawnSync("dmesg", args, { encoding: "utf8", env: { ...process.env, TZ: "UTC", LC_ALL: "C" }, timeout: 2000, maxBuffer: 128 * 1024 }); - return { args, status: probe.status, stderr: probe.stderr, version: args[0] === "--version" ? probe.stdout : undefined }; - }) : []; - expect(["EPERM", "EACCES", "ENOENT", "ENOTSUP", "ETIMEDOUT", "ENOBUFS"], JSON.stringify(debug)).toContain(record.kernel.reason); + expect(["EPERM", "EACCES", "ENOENT", "ENOTSUP", "ETIMEDOUT", "ENOBUFS"]).toContain(record.kernel.reason); } const resultRecord = entry === "planned" ? read(path.join(fixture.dir, "results/shard-1.json")).tasks[0] : read(path.join(fixture.dir, "results/lifecycle.json")); expect(resultRecord.resourceDiagnosticsPaths).toHaveLength(1); From 296391c0038b780d635f7903f395b93f6082962a Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:23:53 +0000 Subject: [PATCH 33/64] Record slice 2 diagnostic proof and Blaxel validation --- .../test-harness-resource-cleanup-plan.md | 4 +- ...arness-resource-cleanup-review-response.md | 10 +++ .../test-harness-resource-cleanup-slice-2.md | 68 +++++++++++++++++++ proofs/arch/test-runner/reporting.md | 8 +-- 4 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 docs/working/test-harness-resource-cleanup-slice-2.md diff --git a/docs/working/test-harness-resource-cleanup-plan.md b/docs/working/test-harness-resource-cleanup-plan.md index 92c0c68d..48b88a6d 100644 --- a/docs/working/test-harness-resource-cleanup-plan.md +++ b/docs/working/test-harness-resource-cleanup-plan.md @@ -1,6 +1,6 @@ # Test-harness resource cleanup: implementation plan -Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; [slice 1 is implemented](test-harness-resource-cleanup-slice-1.md) for independent review. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. +Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; [slice 1 has converged](test-harness-resource-cleanup-slice-1-review-r1.md), and [slice 2 is implemented](test-harness-resource-cleanup-slice-2.md) for independent review. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. ## Slice judgment @@ -93,4 +93,4 @@ Retrieve all evidence before deleting the dedicated coordinator, and report clea ## Current limits -Slice 1's helper contracts, real framework teardown and five built-application walks have red/green evidence, with affected-file and caller checks recorded in its evidence document. The named private coordinator and two workers exist; credentials, Node and x64/Linux execution are established there. Provider inventory differs from the historical pool count, and account-specific quota and idle-sandbox accounting remain unknown. The coordinator's experiment-driver preflight, renewal/recovery and pool-label behavior, interrupted dispatch, browser temporary placement, diagnostic transport and full-gate duration remain for slices 2 and 3. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. +Slice 1's helper contracts, real framework teardown and five built-application walks have converged. Slice 2's bounded collector, parent lifecycle and retained-link contracts have red/green evidence, with affected-file checks and remaining live-provider obligations recorded in its evidence document. The named private coordinator and two workers exist; credentials, Node and x64/Linux execution are established there. Provider inventory differs from the historical pool count, and account-specific quota and idle-sandbox accounting remain unknown. The coordinator's experiment-driver preflight, renewal/recovery and pool-label behavior, interrupted dispatch, browser temporary placement, diagnostic retention across reuse and recovery, and full-gate duration remain for slice 3. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index c9efaec3..9affa90b 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -216,3 +216,13 @@ The supervisor arranges independent review of this slice before further implemen The supervisor relayed **PASS** at `036d0491`; the [complete review](test-harness-resource-cleanup-slice-1-review-r1.md) is preserved unchanged. Apply the proof-only refinement by attributing per-case server disposal to the real seam, and clarify that the final static log ends with the separately recorded fetch interruption. Keep the second framework invocation: it directly exercises the reviewed later-worker lifetime assertion, and removing it now would change that proof's scope. This optional cost reduction does not block convergence. The reported 143 deployed shared sandboxes remain an operator question for Josh and an integrated-review documentation item. Do not infer quota semantics or change the pool. Slice 2 implements bounded parent diagnostics, retained links and retry history; live provider recovery/diagnostic qualification and the lease/download race remain explicitly pending until slice 3. + +## Slice 2: implemented for independent review + +The [slice 2 evidence record](test-harness-resource-cleanup-slice-2.md) records red/green commits, exact canonical selections, all intermediate failures and the remaining boundaries. The final code checkpoint is `0e82df7f`; the following proof/document update changes no executable behavior. + +The shared collector retains a baseline before preparation and final observations after cleanup for planned standalone, combined Vitest and targeted execution. The targeted Vitest path uses the existing attempt recorder so recovered failures trigger kernel capture. Bounded serialization and download preserve identities, unavailable reasons, counter resets, exact execution intervals and native evidence, with no sample payload in compact summaries or timing events. Aggregation keeps earlier attempt links without replacing the final native outcome. The real-reader seam caught an installed util-linux parser incompatibility; the implementation now reads an enclosing whole-second UTC window and filters its output to the exact observation interval. + +Five collector/interval contracts, two retained-link contracts, four parent cases and the combined-project seam have passed. Complete reporting, provider and shard-worker files passed, followed by focused checks for the kernel-reader corrections. Lint and type checking passed on the private coordinator at the final code checkpoint. Every test used the canonical runner and private pool at a pushed revision; none ran on the shared development host or shared pool. All executed runs reported zero process leaks and infrastructure failures. Temporary diagnostic probes used during kernel-reader debugging are removed. + +The reporting proof now cites the implemented bounds, parent and link coverage. Its live download/reuse and recovery-receipt assertion stays pending for slice 3. The current lease/download race, worker recovery, scratch reclamation, persistent-worker retention, full verification and integrated review are still outstanding. The private hosts remain allocated. The supervisor arranges independent review before slice 3 begins. No new durable team decision was made. diff --git a/docs/working/test-harness-resource-cleanup-slice-2.md b/docs/working/test-harness-resource-cleanup-slice-2.md new file mode 100644 index 00000000..aade87cc --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-slice-2.md @@ -0,0 +1,68 @@ +# Test-harness cleanup: slice 2 evidence + +Owner: rose. Draft PR #12, branch `thopter/test-harness-resource-cleanup`. Derived from the converged [plan](test-harness-resource-cleanup-plan.md). This slice implements bounded resource observations and their report transport. Worker exclusion through retrieval, abandoned-run recovery, temporary-directory cleanup and the dedicated live acceptance remain assigned to slice 3. + +## Changes and review scope + +The shared collector runs in the planned task parent and targeted supervisor. It writes a baseline before surface preparation and finalizes after service and owned-process cleanup. Combined Vitest projects receive separate surface records referencing their split native reports and shared attempt recorder. Missing final observations remain explicit after abrupt parent death. Linux probes retain `MemAvailable` and `Shmem` from `/proc/meminfo`, free bytes from `statfs` on the effective temporary directory, and `oom_kill` from `/proc/vmstat`, with units, source and observation time. Missing probes remain unknown; a reset cannot become a negative OOM delta. + +The targeted Vitest path now uses the existing generated runner config and attempt recorder. This supplies actual failed-then-passed attempts to both reporting and diagnostic capture; final JSON alone cannot establish a recovered retry. Diagnostic decisions use the existing native normalizer. A failed native attempt, incomplete evidence or a nonzero exit requests bounded kernel evidence for the execution interval. The kernel reader has a two-second timeout and a 128 KiB read cap; it retains only relevant OOM identities and numeric fields, with at most 64 KiB of lines. It does not retain arbitrary kernel messages or arguments. + +One writer enforces 128 KiB per sidecar and 8 MiB per shard attempt, including recovery receipts. It reserves compact observations for already-created baselines before excerpts consume the budget. Receipt serialization is available for slice 3; this slice does not produce worker recovery receipts. Truncation, omitted data and unavailable readings are explicit. Probe/write failures warn without replacing the native outcome. Neither sample values nor kernel excerpts enter console summaries or timing events. + +The coordinator copies only bounded files matching the current provider run, shard and infrastructure-attempt identity. Diagnostic attempt identity includes the retry number because the current lock ID alone is reused across retries. Provider aggregation retains each attempt's files while using the latest task outcomes; final `resourceDiagnosticsPaths` entries are relative to the canonical run directory and name retained files only. Sidecars keep relative links to their native JSON and attempt NDJSON. Baselines recovered without a task summary remain provider evidence and cannot invent a selected-surface result. + +No production application file, dependency, timeout default or retry default changes. Lease/release behavior is not changed here. The present release-before-download race remains until slice 3; this checkpoint is not safe concurrent-worker-reuse qualification. + +## Placement, red/green and corrections + +All tests used committed, pushed revisions from `/workspace/television` on `tv-cleanup-coordinator-rose`, through the canonical Blaxel runner with private pool `rose-cleanup`. The two execution workers were `tv-testshard-x64-rose-cleanup-01` and `-02`. Nothing was tested, built, linted or type-checked on the shared development host, and no shared worker was touched. All outer selections are `unit:root` with the default zero native retries and two infrastructure retries. The authored nested fixtures deliberately retry to establish actual failed and passed attempt evidence. No isolated-GitHub lifecycle fixture ran. + +- `bda9b6cd` contains the three focused files and a minimal writer placeholder so failures reach assertions rather than missing imports. Four collector contracts failed on absent observations/bounds and an unretained link; two link contracts failed on absent path/history propagation; four real parent cases failed on absent baselines. The native fixtures reached their intended pass, early exit or interruption paths. These are red evidence, with no infrastructure failure or leaked process. +- `e4a094a7` adds diagnostic assertions to the existing real combined-project seam. It failed on missing paths after correctly recording one passing member and one recovered member. +- `24229bd3` implements the collector and transport. Collector contracts and the combined-project case passed. The link test reached its timing-event assertion and exposed an authored fixture typo (`id` instead of `runId`); the interruption test exposed that SIGTERM legitimately lets the supervisor finish and retain a final sample. These were test defects, not evidence that those production paths should change. +- `86bdf6da` corrects the timing fixture and makes the tiny preparation fixture SIGKILL its own parent, then exit itself. The parent test waits for that signal and disposes its owned handle. This proves missing final evidence without leaving a descendant. The budget contract also finalizes a second baseline after receipts saturate the attempt budget. All three focused files then passed, followed by complete reporting, provider and shard-worker files. Lint and type checking passed on the coordinator; the log is retained at `.test-runs/slice-2-static/86bdf6da.log`. +- `83e8c9f1` stops labelling every nonzero kernel-reader exit as a permission error. Permission reasons now require corresponding command evidence; other failures remain unknown. Collector and parent files passed. `f72654e1` distinguishes diagnostic infrastructure attempts and passed the provider file through real download. +- Final inspection found that accepting any unknown kernel-reader error could conceal a malformed invocation. The [util-linux calendar parser](https://github.com/util-linux/util-linux/blob/v2.39.3/lib/timeutils.c) does not accept the trailing `Z` of JavaScript ISO timestamps. The stricter real-reader assertion at `0db76dd5` failed in both planned and targeted cases with the generic `unavailable` reason. `cf4fb510` tried epoch seconds; the installed worker parser rejected those too. A narrowed diagnostic revision, `734a29ac`, retained `dmesg: invalid time value "@1793124854.151"` in the native assertion. `dcc0847c` tried fractional calendar bounds, which also failed. The narrowed `056f9e21` probe established util-linux 2.38.1 on the worker: whole-second calendar times returned status 0; both fractional calendar and epoch times were rejected. The final reader uses an enclosing whole-second UTC window, then filters the returned ISO timestamps to the exact observation interval. `b9e9cbf8` first adds a red contract for that filtering, including disclosed omission of OOM lines without timestamps. The temporary command probes are removed from the final seam. At `0e82df7f`, all five collector contracts and all four parent cases passed; lint and type checking also passed on the private coordinator. The final static log is retained at `.test-runs/slice-2-static/0e82df7f.log`. + +## Canonical run evidence + +Each selection below follows `npm test -- blaxel --pool rose-cleanup`. The orchestration-only `--run-dir-output` receipt does not change selection. Run directories exist under `/workspace/television/.test-runs/` on the coordinator and under `.test-runs/` in this worktree. Retained files include canonical summary/results/events, provider request/report/setup/shard/log and native JSON. Downloaded diagnostic files and their referenced native/attempt files are retained locally for the inspected runs. Every listed run has zero process leaks and zero infrastructure failures; outer passing runs have zero recovered retries. Nested retry evidence is checked by the real seams rather than counted as an outer flaky test. + +| Commit | Canonical selection | Result | Run-directory name | +| --- | --- | --- | --- | +| `bda9b6cd` | `--file test/repo/resource-diagnostics.test.ts` | failed (4 failing cases) | `2026-10-27T17-47-54-943Z-p10651-r3ac5350dc8a133e8` | +| `bda9b6cd` | `--file test/repo/resource-diagnostics-links.test.ts` | failed (2 failing cases) | `2026-10-27T17-48-04-531Z-p10744-r14c37ae4adc16ed0` | +| `bda9b6cd` | `--file test/repo/test-runner-resource-parent.test.ts` | failed (4 failing cases) | `2026-10-27T17-48-14-171Z-p10838-r3883e2af32ea2e2a` | +| `e4a094a7` | `--file test/repo/test-runner-shard-worker.test.ts --grep 'runs exact files from two Vitest projects in one process'` | failed (1 failing case) | `2026-10-27T17-59-15-407Z-p11411-rbae49973b433bf36` | +| `24229bd3` | `--file test/repo/resource-diagnostics.test.ts` | passed | `2026-10-27T17-59-49-606Z-p11918-r8fc8bc0fe0c4d379` | +| `24229bd3` | `--file test/repo/resource-diagnostics-links.test.ts` | failed (1 failing case) | `2026-10-27T17-59-57-947Z-p12010-r834336b8375e5b32` | +| `24229bd3` | `--file test/repo/test-runner-resource-parent.test.ts` | failed (1 failing case) | `2026-10-27T18-00-05-937Z-p12102-r5e012a44da71b58e` | +| `24229bd3` | `--file test/repo/test-runner-shard-worker.test.ts --grep 'runs exact files from two Vitest projects in one process'` | passed | `2026-10-27T18-00-13-973Z-p12194-r282952d31fc443ed` | +| `86bdf6da` | `--file test/repo/resource-diagnostics.test.ts` | passed | `2026-10-27T18-02-11-101Z-p12741-r418e495d464a8309` | +| `86bdf6da` | `--file test/repo/resource-diagnostics-links.test.ts` | passed | `2026-10-27T18-02-19-656Z-p12834-rfdb3a34bd4c461f6` | +| `86bdf6da` | `--file test/repo/test-runner-resource-parent.test.ts` | passed | `2026-10-27T18-02-28-420Z-p12926-r60d6db28b6e495cf` | +| `86bdf6da` | `--file test/repo/test-runner-reporting.test.ts` | passed | `2026-10-27T18-03-27-138Z-p13498-r6e3b77fbc4738fcd` | +| `86bdf6da` | `--file test/repo/test-runner-provider.test.ts` | passed | `2026-10-27T18-03-40-246Z-p13595-r2c5e2ec7f7595f57` | +| `86bdf6da` | `--file test/repo/test-runner-shard-worker.test.ts` | passed | `2026-10-27T18-03-48-465Z-p13687-r720be13bc8846ae1` | +| `83e8c9f1` | `--file test/repo/resource-diagnostics.test.ts` | passed | `2026-10-27T18-05-26-980Z-p14233-r1e947b2039364e32` | +| `83e8c9f1` | `--file test/repo/test-runner-resource-parent.test.ts` | passed | `2026-10-27T18-05-36-797Z-p14326-r2d359f1f9dcc8ed7` | +| `f72654e1` | `--file test/repo/test-runner-provider.test.ts` | passed | `2026-10-27T18-07-08-959Z-p14854-r758126113ffe4f59` | +| `0db76dd5` | `--file test/repo/test-runner-resource-parent.test.ts` | failed (2 failing cases) | `2026-10-27T18-10-50-295Z-p15426-reb1cb3404f6eb3af` | +| `cf4fb510` | `--file test/repo/test-runner-resource-parent.test.ts` | failed (2 failing cases) | `2026-10-27T18-11-37-879Z-p15942-rd69720c74d517af3` | +| `734a29ac` | `--file test/repo/test-runner-resource-parent.test.ts --grep 'targeted parent retains'` | failed (1 failing case) | `2026-10-27T18-14-10-644Z-p16480-rf4c2e4c1ed35a761` | +| `dcc0847c` | `--file test/repo/test-runner-resource-parent.test.ts` | failed (2 failing cases) | `2026-10-27T18-15-40-396Z-p17010-r22c8011ef51efab8` | +| `056f9e21` | `--file test/repo/test-runner-resource-parent.test.ts --grep 'targeted parent retains'` | failed (1 failing case) | `2026-10-27T18-17-26-697Z-p17541-rf5de936fb50934b4` | +| `b9e9cbf8` | `--file test/repo/resource-diagnostics.test.ts --grep 'restricts a whole-second'` | failed (1 failing case) | `2026-10-27T18-19-53-507Z-p18065-r37c3f452fa9ba8bf` | +| `0e82df7f` | `--file test/repo/resource-diagnostics.test.ts` | passed | `2026-10-27T18-20-55-154Z-p18587-r0f555091be0582c5` | +| `0e82df7f` | `--file test/repo/test-runner-resource-parent.test.ts` | passed | `2026-10-27T18-21-03-544Z-p18679-r62fcc5d3e0807a57` | + +## Proof coverage and limits + +The [reporting proof](../../proofs/arch/test-runner/reporting.md) now cites the five collector/interval contracts, two link/history contracts and four real parent cases, plus the existing combined-project seam. The contracts use exactly the declared probe/clock/write substitutions and recorded native fixture. The parent seams use real Linux probes, preparation, installed Vitest, its attempt recorder and teardown. The abrupt-parent case has no replacement reporter or invented native result. The whole reporting file also exercises the established native-recorder crossing. + +An inspected successful coordinator download at `f72654e1` retained a 1,605-byte sidecar with provider run ID, shard 1, infrastructure attempt 1, surface `unit:root`, real start/end values, and resolvable JSON/NDJSON references. The identity maps to `provider/blaxel/request.json`'s run ID; the outer canonical run retains that request. This is useful transport evidence, not the dedicated planned/targeted reuse or recovery acceptance. Start/end values are not peak usage, and unavailable kernel evidence cannot exclude OOM. No OOM or resource exhaustion was induced. + +`^reporting-t-resource-live` stays pending. Slice 3 must qualify download under held ownership, subsequent cleanup/reuse, abandoned-run receipts, persistent-worker retention and actual Chromium/Node temporary placement. Full verification, integrated review, human spec acceptance and the shared-pool inventory question also remain outstanding. The private coordinator and both workers remain allocated for the next slice. + +Primitive's commit hook surfaced a truncated set of existing decisions; it was not a complete constraint check. No decision was overridden or reconciled. This slice implements the converged direction and establishes no new durable team policy. diff --git a/proofs/arch/test-runner/reporting.md b/proofs/arch/test-runner/reporting.md index 858e212c..929f59ca 100644 --- a/proofs/arch/test-runner/reporting.md +++ b/proofs/arch/test-runner/reporting.md @@ -10,7 +10,7 @@ Native-report contracts carry normalization breadth; real native-runner and CLI Resource diagnostics compose a bounded writer contract, a real Linux probe/parent-lifecycle seam, and the dedicated Blaxel acceptances owned by [the provider proof](blaxel-testshards.md#Lease, recovery, and scratch). A small native failure and a recovered retry establish retained evidence; they do not reproduce resource exhaustion. Authored unavailable/reset/oversized probe inputs cover uncommon diagnostic failures without filling a disk or causing an OOM kill. Those mocks forfeit kernel reading and transport to the real seams. -Planned and targeted execution both cross parent capture and download. The same sidecars pass through provider normalization and attempt history; a recovered infrastructure attempt must not replace earlier evidence. Recovery receipts are observed in the provider's crash acceptance, not a duplicate crash here. No test claims start/end samples measure a peak or that an unavailable kernel log excludes OOM. The new resource assertions remain pending until implemented and run. +Planned and targeted execution share the collector; the parent seams cover capture and the dedicated Blaxel acceptances cover download. The same sidecars pass through provider normalization and attempt history; a recovered infrastructure attempt must not replace earlier evidence. Recovery receipts are observed in the provider's crash acceptance, not a duplicate crash here. No test claims start/end samples measure a peak or that an unavailable kernel log excludes OOM. The bounds, parent capture and retained-link assertions have passed on Blaxel; the dedicated live download/reuse and recovery-receipt acceptance remains pending in slice 3. ## Test hooks @@ -48,10 +48,10 @@ The event-normalization assertions are **contract tests** over recorded native-r ### Resource diagnostics -- **Contract — bounded, honest observations.** The collector/writer consumes authored start/end memory, filesystem, OOM-counter and kernel-line readings, including unavailable probes, a counter reset and over-budget excerpts, and writes real files in a temporary results root. Under [resource diagnostics](../../../specs/arch/test-runner/reporting.md#^resource-diagnostics) and [bounds](../../../specs/arch/test-runner/reporting.md#^resource-diagnostic-bounds), values keep units/time, unknowns keep reasons rather than zero, and resets produce no negative OOM delta. A failed native attempt, even if later recovered, or incomplete execution requests the interval's kernel evidence. The bounded reader/writer retains at most 64 KiB of kernel lines and 128 KiB per sidecar, and 8 MiB total per shard attempt including receipts; compact start/end evidence survives before optional excerpts, with explicit truncation. Only allowed fields are serialized: supplied credential, query/body, environment and argument sentinels do not appear. Injected probe and write failures warn without changing the original native outcome. Mock readings/write failures forfeit OS reading to the parent seam and actual download to the live assertion below. (test to be written) ^reporting-t-resource-bounds +- **Contract — bounded, honest observations.** The collector/writer consumes authored start/end memory, filesystem, OOM-counter and kernel-line readings, including unavailable probes, a counter reset and over-budget excerpts, and writes real files in a temporary results root. Under [resource diagnostics](../../../specs/arch/test-runner/reporting.md#^resource-diagnostics) and [bounds](../../../specs/arch/test-runner/reporting.md#^resource-diagnostic-bounds), values keep units/time, unknowns keep reasons rather than zero, and resets produce no negative OOM delta. A failed native attempt, even if later recovered, or incomplete execution requests the interval's kernel evidence. Authored timestamped lines prove that a whole-second read is filtered to the exact observation interval and untimestamped OOM lines are counted as omitted. The bounded reader/writer retains at most 64 KiB of kernel lines and 128 KiB per sidecar, and 8 MiB total per shard attempt including receipts; compact start/end evidence survives before optional excerpts, with explicit truncation. Only allowed fields are serialized: supplied credential, query/body, environment and argument sentinels do not appear. Injected probe and write failures warn without changing the original native outcome. Mock readings/write failures forfeit OS reading to the parent seam and actual download to the live assertion below. ([`test/repo/resource-diagnostics.test.ts`](../../../test/repo/resource-diagnostics.test.ts)) ^reporting-t-resource-bounds -- **Seam — parent capture survives the child.** The production surface parent crosses real Linux procfs/filesystem probes and an installed native runner's child-process boundary in a disposable results root. Authored tiny native fixture tests produce a failure/retry and an exit before native output; both planned-worker and targeted execution wiring are exercised. The retained baseline exists before preparation begins and the final observation follows teardown; a terminated child cannot erase the baseline. A separate interrupted-parent fixture leaves the baseline with an explicitly missing final observation rather than invented zeroes. Real probe values are checked for their source/units and availability reasons, not exact memory totals. No kernel OOM is manufactured; inaccessible kernel evidence must remain unknown. No mechanisms are mocked. This proves the capture boundary in [resource diagnostics](../../../specs/arch/test-runner/reporting.md#^resource-diagnostics); the live assertion carries remote retrieval. (test to be written) ^reporting-t-resource-parent +- **Seam — parent capture survives the child.** The production surface parent crosses real Linux procfs/filesystem probes and an installed native runner's child-process boundary in a disposable results root. Authored tiny native fixture tests produce a failure/retry and an exit before native output; both planned-worker and targeted execution wiring are exercised. The retained baseline exists before preparation begins and the final observation follows teardown; a terminated child cannot erase the baseline. A separate interrupted-parent fixture leaves the baseline with an explicitly missing final observation rather than invented zeroes. Real probe values are checked for their source/units and availability reasons, not exact memory totals. No kernel OOM is manufactured; inaccessible kernel evidence must remain unknown. No mechanisms are mocked. This proves the capture boundary in [resource diagnostics](../../../specs/arch/test-runner/reporting.md#^resource-diagnostics); the live assertion carries remote retrieval. ([`test/repo/test-runner-resource-parent.test.ts`](../../../test/repo/test-runner-resource-parent.test.ts); combined-project member attribution in [`test/repo/test-runner-shard-worker.test.ts`](../../../test/repo/test-runner-shard-worker.test.ts), “runs exact files from two Vitest projects in one process”) ^reporting-t-resource-parent -- **Contract — retained links and attempt history.** Provider normalization and infrastructure-attempt aggregation consume authored shard/task envelopes referring to real temporary sidecar files, with recorded native failed/recovered fixtures. Under [normalized results](../../../specs/arch/test-runner/reporting.md#Normalized surface result) and [resource diagnostics](../../../specs/arch/test-runner/reporting.md#Resource diagnostics), each advertised `resourceDiagnosticsPaths` entry exists relative to the final run directory; earlier attempt evidence survives a later successful infrastructure attempt. Missing native output remains incomplete and cannot manufacture a selected-surface result. Samples and kernel excerpts appear in neither compact console output nor timing events. Mocked provider envelopes forfeit remote creation/download to the live assertion; recorded native inputs forfeit attempt capture to the real parent seam and existing native-recorder assertions above. (test to be written) ^reporting-t-resource-links +- **Contract — retained links and attempt history.** Provider normalization and infrastructure-attempt aggregation consume authored shard/task envelopes referring to real temporary sidecar files, with recorded native failed/recovered fixtures. Under [normalized results](../../../specs/arch/test-runner/reporting.md#Normalized surface result) and [resource diagnostics](../../../specs/arch/test-runner/reporting.md#Resource diagnostics), each advertised `resourceDiagnosticsPaths` entry exists relative to the final run directory; earlier attempt evidence survives a later successful infrastructure attempt. Missing native output remains incomplete and cannot manufacture a selected-surface result. Samples and kernel excerpts appear in neither compact console output nor timing events. Mocked provider envelopes forfeit remote creation/download to the live assertion; recorded native inputs forfeit attempt capture to the real parent seam and existing native-recorder assertions above. ([`test/repo/resource-diagnostics-links.test.ts`](../../../test/repo/resource-diagnostics-links.test.ts)) ^reporting-t-resource-links - **Acceptance — evidence reaches the caller.** Share the dedicated-pool canonical CLI runs in [the provider retrieval acceptance](blaxel-testshards.md#^blaxel-t-retrieval), under its shared `experiment:blaxel-harness` declaration and dedicated coordinator placement. Its `experiment:blaxel-harness-fixtures` selection includes a committed fail-once native fixture that recovers under the default retry budget and a fixture that fails all attempts. Planned and targeted runs retain the native outcomes and their start/end sidecars after worker scratch is removed; every normalized diagnostic link opens a downloaded file with the correct run/shard/surface identity and native-attempt references. The failed run still fails, the recovered case retains both attempts, and any inaccessible kernel evidence says so. The provider's [crash acceptance](blaxel-testshards.md#^blaxel-t-recovery-live) supplies the downloaded recovery receipt and its disclosed report loss. Boundary: real CLI, Blaxel API, native runners and report files, with the provider proof's scheduling hook and no mocks. This proves [resource retention](../../../specs/arch/test-runner/reporting.md#Resource diagnostics) end to end without a resource-exhaustion experiment. (test to be written) ^reporting-t-resource-live From 1320c9573d34e22f18f2c4537fd44b463a123ca7 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:40:49 +0000 Subject: [PATCH 34/64] test: expose Blaxel lease recovery gaps --- ...arness-resource-cleanup-review-response.md | 14 +++++ ...ness-resource-cleanup-slice-2-review-r1.md | 57 +++++++++++++++++++ scripts/run-blaxel-testshards.mjs | 18 +----- scripts/test/blaxel-worker-lease.mjs | 17 ++++++ test/repo/blaxel-worker-lease.test.ts | 38 +++++++++++++ 5 files changed, 128 insertions(+), 16 deletions(-) create mode 100644 docs/working/test-harness-resource-cleanup-slice-2-review-r1.md create mode 100644 scripts/test/blaxel-worker-lease.mjs create mode 100644 test/repo/blaxel-worker-lease.test.ts diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 9affa90b..05442aee 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -226,3 +226,17 @@ The shared collector retains a baseline before preparation and final observation Five collector/interval contracts, two retained-link contracts, four parent cases and the combined-project seam have passed. Complete reporting, provider and shard-worker files passed, followed by focused checks for the kernel-reader corrections. Lint and type checking passed on the private coordinator at the final code checkpoint. Every test used the canonical runner and private pool at a pushed revision; none ran on the shared development host or shared pool. All executed runs reported zero process leaks and infrastructure failures. Temporary diagnostic probes used during kernel-reader debugging are removed. The reporting proof now cites the implemented bounds, parent and link coverage. Its live download/reuse and recovery-receipt assertion stays pending for slice 3. The current lease/download race, worker recovery, scratch reclamation, persistent-worker retention, full verification and integrated review are still outstanding. The private hosts remain allocated. The supervisor arranges independent review before slice 3 begins. No new durable team decision was made. +# Slice 2 convergence and slice 3 start + +The supervisor relayed PASS at `296391c0`; the complete review is preserved in +`test-harness-resource-cleanup-slice-2-review-r1.md`. Accept the bounded +refinements: reuse the timing-provider constant, resolve targeted Vitest configs +from the declared surface cwd, and reconcile recovery receipts with slice 3's +actual records. Passing surfaces intentionally retain compact start/end sidecars +within the existing per-attempt budget; this aids comparisons without a sampler +or cross-run archive. + +Slice 3 starts with real generated lock-command checks on disposable Linux roots. +The first checkpoint only extracts the existing command without changing its +timestamp-only behavior; the new assertions require renewable ownership and +confirmed quiet-orphan recovery. Execution is pending on the private Blaxel pool. diff --git a/docs/working/test-harness-resource-cleanup-slice-2-review-r1.md b/docs/working/test-harness-resource-cleanup-slice-2-review-r1.md new file mode 100644 index 00000000..5169ff09 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-slice-2-review-r1.md @@ -0,0 +1,57 @@ +# Slice 2 implementation review, round 1: test-harness resource cleanup (PR #12, 296391c0) + +**Verdict: PASS.** Slice 2 delivers the diagnostics the reporting spec promises and no more: four Linux readings with units, source and time at start and end of every Blaxel surface execution, written before preparation and finalized after owned-process cleanup on both the planned and the targeted path; unknowns kept as reasons, never zero; a counter reset that cannot become a negative delta; kernel OOM evidence read only when a native attempt failed or evidence is incomplete, through a two-second, 128 KiB-capped `dmesg` window filtered to the exact interval, retaining only process identity and numeric fields; one writer that enforces 64 KiB of kernel lines, 128 KiB per sidecar and 8 MiB per shard attempt including receipts, with compact observations reserved before excerpts; links that name only retained files, relative to the run directory, kept across infrastructure retries; and a coordinator download that accepts only files matching the run, shard and attempt it dispatched. The three proof assertions assigned to this slice are discharged by tests at the declared shapes, with the declared substitutes and nothing else mocked, and the record's evidence checks out row by row. No production file changed. Three refinements and one observation follow, none blocking. + +- Reviewed: `git diff 036d0491 296391c0` in full (the new module and types; coordinator, planned worker, supervised command, reporting, provider normalization and outputs; three new tests, three fixtures, the combined-project seam extension; the proof regrades; the record); against the plan's slice 2, `^resource-diagnostics` and `^resource-diagnostic-bounds`, the three proof assertions, the testing policy and the reviewer checklist; plus the retained run directories and static logs in this worktree. +- Branch state: contains `origin/main` (`a8e73fbf`); the copied slice 1 review is identical to mine; the response applies my proof refinement, declines the optional cost cut with a stated reason, and adds the static-log note. +- I ran nothing. I read summaries, one downloaded sidecar and two logs with small scripts; nothing was built, tested or changed. + +## What the slice proves, against the proofs + +**`^reporting-t-resource-bounds` (contract, `test/repo/resource-diagnostics.test.ts`, five cases).** With authored probe readings, a controlled clock and an injectable write, in a real temporary root: a baseline file exists before finalization with `end.status: "missing"`; an `EACCES` probe is kept as `value: null` with its reason; a counter that goes 10 → 2 yields `oomDelta.reason: "counter-reset"`; a recorded Playwright retry fixture (`test/repo/fixtures/test-runner/playwright-retry.json`) is classified `failedAttempt: true, recovered: true` and triggers a kernel request for exactly the start/end interval; a secret planted in identity, probe errors and a kernel line never reaches the file or the warnings; all-`EPERM` probes leave every value unknown with `kernel.reason: "EPERM"`; 8,000 kernel lines truncate to at most 64 KiB inside a file of at most 128 KiB; eighty 3,000-process receipts saturate the attempt budget and a pending baseline still gets its compact final observation; an `ENOSPC` write warns and returns no link; and a whole-second read is filtered to the exact interval with untimestamped OOM lines counted as omitted. That is the contract's text, clause by clause. + +**`^reporting-t-resource-parent` (seam, `test/repo/test-runner-resource-parent.test.ts`, four cases, plus the combined-project case in `test-runner-shard-worker.test.ts`).** The real planned worker (`scripts/run-test-shard.mjs --plan`) and the real targeted supervisor (`scripts/test/supervised-command.mjs`) each run an authored Vitest fixture under installed Vitest with `retry: 2`. The pre-command (`prepare.mjs`) reads the results tree during preparation and records what it finds, which is how the test proves the baseline existed, with its end missing, before preparation; the fixture's `afterAll` records teardown time, and the final observation is later than it. Each of the four probes is checked for source and units, with real values only required to be non-negative or carry a reason. The attempt file shows `["failed","passed"]`, so the recovered retry is real. Kernel status must be `available` or `unavailable` with an attributable code; the generic `unavailable` that a malformed invocation produces is refused, which is exactly how this seam caught the installed util-linux parser rejecting the first three timestamp formats (`0db76dd5` through `056f9e21`). The exit fixture (`setup.mjs` exits 17 before any test) leaves `end: observed, nativeEvidence: missing`; the interrupt fixture `SIGKILL`s its own supervising parent and the baseline survives with `end.status: "missing", reason: "parent-did-not-finalize"`, with the owned handle disposed and no survivor. The combined Vitest seam now checks one sidecar per project member with the right identity and a `resultPath` that resolves to that member's split native report. Nothing is mocked; the only hook is the guarded registry module, used by the local child and, as the proof hooks now say, never forwarded to workers. + +**`^reporting-t-resource-links` (contract, `test/repo/resource-diagnostics-links.test.ts`).** Two authored infrastructure attempts with real sidecar files: the superseded attempt's file survives in the history, the final surface links both attempts' files and drops a missing one, the recovered flake and clean failure list come from the recorded native fixture, every link resolves relative to the run directory, and neither `summary.json`, the compact provider summary nor `events.ndjson` contains a sample value or a kernel sentinel while the sidecar still does. A baseline-only provider failure normalizes to `provider:blaxel`, `incomplete`, `testStatus: unknown`, zero surfaces run, with the baseline linked. + +**The implementation behind them.** `startResourceDiagnostics` writes the baseline at construction and rewrites the same file at `finish`, deciding the kernel request from exit code, incompleteness, native status and failed attempts (`scripts/test/resource-diagnostics.mjs:31-60`); `readLinuxResourceProbe` reads `MemAvailable` and `Shmem` from `/proc/meminfo` (kB to bytes), `statfs` on the effective temporary directory and `oom_kill` from `/proc/vmstat`, and runs `dmesg --kernel --time-format iso` over an enclosing whole-second UTC window with a 2 s timeout and 128 KiB buffer, mapping `ENOBUFS` to truncation and permission text to `EPERM`/`EACCES` (`:79-102`); `kernelLinesInInterval` and `cleanKernel` keep only timestamped lines carrying `Killed process ()` and numeric OOM fields (`:110-122`, `:145-163`); `writeResourceDiagnostic` whitelists fields, reserves 8 KiB per existing file, binary-searches the excerpt to the budget and writes atomically (`:166-202`). The planned worker starts one collector per task before pre-commands and finishes it in the `finally` after `supervisor.finish()`, marking incompleteness on unconfirmed cleanup, failed pre-commands, service failure or leaks; the supervised command does the same around its whole run and, for Vitest targets, wraps the config with the attempt reporter so recovered retries are real evidence there too. The coordinator exports run, shard and `-attempt-` identity to the worker, lists `.testshards/results/resources`, caps the listing, refuses any file over 128 KiB or beyond 8 MiB in total or whose identity is not this run, shard and attempt, and copies the rest under `artifacts/shard-/resources/` on both the success and the failure path. Aggregation keeps every attempt's links while taking the latest outcomes; `retainedResourcePaths` keeps only files that exist, by exact path. + +## The record, checked + +| Claim | Evidence | +| --- | --- | +| 25 runs, each at the named commit with the named result; zero recovered retries, leaks and infrastructure failures | All 25 summaries present; `git.commit`, outcome and failure counts agree with every row; `testsFlakyRecovered`, `processLeaks` and `infraFailures` are zero throughout. | +| Lint and type-check passed at `86bdf6da` and `0e82df7f` | `.test-runs/slice-2-static/86bdf6da.log` and `0e82df7f.log` show both commands clean (the latter after its two test runs). | +| An inspected download at `f72654e1` retained a 1,605-byte sidecar with provider run ID, shard 1, attempt 1, `unit:root`, real values and resolvable references | `artifacts/shard-1/resources/surface-….json` is 1,605 bytes; `identity.runId` equals `request.json`'s run ID; `attemptId` is `-shard-1-attempt-1`; `resultPath` and `attemptResultPath` resolve; `kernel: not-requested`; `oomDelta: 0`; `results.json` links the run-relative path. | +| The kernel-reader corrections were test-driven on the worker | Six red rows between `0db76dd5` and `b9e9cbf8`, each with the stated failing count; the probes added for diagnosis are gone from the final tree. | +| Default budgets, private pool, nothing on the shared host | Every coordinator line in the logs names `--pool rose-cleanup` and `--target-retries 0` for `unit:root`. | + +## Are the diagnostics as small as the proposal intends? + +Yes. One JSON file per surface execution of about 1.6 KB when nothing failed; a kernel excerpt only on failure or incomplete evidence, capped at 64 KiB; 128 KiB per file and 8 MiB per attempt, enforced at write and again at download; no sampler, process inventory, heap snapshot, archive or daemon; samples kept out of the timing stream and console. The one growth beyond the proposal's minimum is that passing surfaces also keep their start/end file (observation below). + +## Refinements (none blocking) + +1. **Import the provider constant instead of copying it.** `resource-diagnostics.mjs:22` enables diagnostics only when `TV_TEST_TIMING_PROVIDER` equals a literal that `timing-baseline.mjs:8` already exports as `BASELINE_TIMING_PROVIDER`. A renamed worker tier would then switch diagnostics off silently. Importing the constant, or gating on an explicit variable the shard script sets, removes the drift. +2. **The targeted config path assumes a repository-rooted cwd.** The shard script exports `TV_TARGET_CONFIG` as `/workspace/television/`, but the CLI hands the coordinator a config path relative to the surface's `cwd` (`scripts/test/cli.mjs:810-812`). Correct today, because no Vitest surface declares a `cwd`, and the targeted runs in the record exercised it; a package-rooted Vitest surface would get a wrong path. Join with the target cwd, or say the assumption in a comment. +3. **Treat the receipt shape as provisional.** `cleanRecord` already serializes `kind: "recovery"` with the fields the spec names, and the budget contract exercises it, but no recovery data exists yet. Slice 3 should reconcile the shape against what recovery actually knows rather than inherit it. + +**Observation.** Sidecars are retained for passing surfaces as well as failing ones. The spec's retention sentence lists the cases that must be retained and does not forbid this; the attempt budget bounds it; and the baseline on a passing run is the context the proposal wanted. Say the choice in the record so the integrated review does not read it as drift. + +## Reviewer checklist + +- **Proof coverage.** The three assertions are discharged by the cited tests at the declared shapes; the live acceptance stays pending for slice 3 as planned. +- **Mocking and boundaries.** Contracts use only the probe reader, clock and write substitutions the proof hooks declare; the seam and the combined-project case have none; the native fixture is the declared recorded one. +- **Honesty of evidence.** The record distinguishes placeholder red, test-defect red, reader-incompatibility red and green; it claims no OOM reproduction and no live download qualification. +- **Scope.** Runner scripts, tests, fixtures, proofs and working documents; no production, dependency, timeout or retry change; the lease/download race is untouched and stated as pending. +- **Baseline.** Green for its obligations; `^reporting-t-resource-live` pending; the plan's slice 2 baseline statement holds. + +## Evidence and limits + +- Read every changed file; cross-checked all 25 run summaries, the cited provider run's sidecar, links and request, and both static logs; traced the baseline and finalization points in both execution paths and the identity from shard script to download. +- Not established: whether `dmesg` returns lines on the private workers under a real failure (the final parent runs accepted either outcome with an attributable reason; the record's probe at `056f9e21` shows the reader exits 0 there); the diagnostics' behavior across worker reuse and recovery, which is slice 3's. + +## Housekeeping + +- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel beyond scripts that read files already on disk. `git fetch origin` updated remote-tracking refs. +- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index 71fc7aaa..7d0c4621 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -14,6 +14,7 @@ import { RESOURCE_SIDECAR_BYTES, RESOURCE_ATTEMPT_BYTES } from "./test/resource- import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; +import { workerLeaseCommand } from "./test/blaxel-worker-lease.mjs"; import path from "node:path"; import process from "node:process"; @@ -338,7 +339,7 @@ async function acquireAny(sandboxes, { shardIndex, slot, requestedWorkers, verif const expiresAt = Math.floor(Date.now() / 1000) + timeouts.lockTtlSeconds; for (const sandbox of acquisitionOrder(sandboxes, { slot, requestedWorkers })) { if (sandbox.__leased) continue; - const command = `bash -lc ${shellQuote(lockCommand({ lockId, shardIndex, expiresAt }))}`; + const command = `bash -lc ${shellQuote(workerLeaseCommand({ lockId, runId, shardIndex, shardTotal, owner: `${os.hostname()}:${process.pid}`, expiresAt }))}`; const proc = await sandbox.process.exec({ name: `tv-lock-${lockId}`, command, waitForCompletion: true, timeout: 30 }); if (proc.exitCode === 0) { let retainLock = false; @@ -395,21 +396,6 @@ function acquisitionOrder(sandboxes, { slot, requestedWorkers }) { return ordered; } -function lockCommand({ lockId, shardIndex, expiresAt }) { - const owner = `${os.hostname()}:${process.pid}`; - const freshJson = JSON.stringify({ lockId, runId, shard: `${shardIndex}/${shardTotal}`, owner, expiresAt }); - const reclaimedJson = JSON.stringify({ lockId, runId, shard: `${shardIndex}/${shardTotal}`, owner, expiresAt, reclaimed: true }); - return [ - "set -euo pipefail", - "lock=/tmp/tv-testshard.lock", - "now=$(date +%s)", - `if mkdir "$lock" 2>/dev/null; then node -e ${shellQuote(`const fs=require("fs"); const data=${JSON.stringify(freshJson)}; const obj=JSON.parse(data); obj.acquiredAt=Number(process.env.now||${Date.now()}); fs.writeFileSync("/tmp/tv-testshard.lock/owner.json", JSON.stringify(obj));`)}; exit 0; fi`, - "expires=0", - "if [ -f \"$lock/owner.json\" ]; then expires=$(node -e 'try{console.log(JSON.parse(require(\"fs\").readFileSync(process.argv[1],\"utf8\")).expiresAt||0)}catch{console.log(0)}' \"$lock/owner.json\"); fi", - `if [ "$expires" -gt 0 ] && [ "$expires" -lt "$now" ]; then rm -rf "$lock"; mkdir "$lock"; node -e ${shellQuote(`const fs=require("fs"); const data=${JSON.stringify(reclaimedJson)}; const obj=JSON.parse(data); obj.acquiredAt=Number(process.env.now||${Date.now()}); fs.writeFileSync("/tmp/tv-testshard.lock/owner.json", JSON.stringify(obj));`)}; exit 0; fi`, - "exit 42", - ].join("; "); -} async function runShard(lease, outputDir, { attempt, onExecutionRequested = () => {}, onExecutionStarted = () => {}, onResultProgress = () => {} } = {}) { const { sandbox, shardIndex, lockId } = lease; diff --git a/scripts/test/blaxel-worker-lease.mjs b/scripts/test/blaxel-worker-lease.mjs new file mode 100644 index 00000000..f75a33ba --- /dev/null +++ b/scripts/test/blaxel-worker-lease.mjs @@ -0,0 +1,17 @@ +export function workerLeaseCommand({ lockId, runId, shardIndex, shardTotal, owner, expiresAt, tmpRoot = "/tmp" }) { + const freshJson = JSON.stringify({ lockId, runId, shard: `${shardIndex}/${shardTotal}`, owner, expiresAt }); + const reclaimedJson = JSON.stringify({ lockId, runId, shard: `${shardIndex}/${shardTotal}`, owner, expiresAt, reclaimed: true }); + const command = [ + "set -euo pipefail", + "lock=/tmp/tv-testshard.lock", + "now=$(date +%s)", + `if mkdir "$lock" 2>/dev/null; then node -e ${shellQuote(`const fs=require("fs"); const data=${JSON.stringify(freshJson)}; const obj=JSON.parse(data); obj.acquiredAt=Number(process.env.now||${Date.now()}); fs.writeFileSync("/tmp/tv-testshard.lock/owner.json", JSON.stringify(obj));`)}; exit 0; fi`, + "expires=0", + "if [ -f \"$lock/owner.json\" ]; then expires=$(node -e 'try{console.log(JSON.parse(require(\"fs\").readFileSync(process.argv[1],\"utf8\")).expiresAt||0)}catch{console.log(0)}' \"$lock/owner.json\"); fi", + `if [ "$expires" -gt 0 ] && [ "$expires" -lt "$now" ]; then rm -rf "$lock"; mkdir "$lock"; node -e ${shellQuote(`const fs=require("fs"); const data=${JSON.stringify(reclaimedJson)}; const obj=JSON.parse(data); obj.acquiredAt=Number(process.env.now||${Date.now()}); fs.writeFileSync("/tmp/tv-testshard.lock/owner.json", JSON.stringify(obj));`)}; exit 0; fi`, + "exit 42", + ].join("; "); + return command.replaceAll("/tmp/tv-testshard.lock", `${tmpRoot}/tv-testshard.lock`); +} + +function shellQuote(value) { return `'${String(value).replaceAll("'", `'"'"'`)}'`; } diff --git a/test/repo/blaxel-worker-lease.test.ts b/test/repo/blaxel-worker-lease.test.ts new file mode 100644 index 00000000..3e6803d7 --- /dev/null +++ b/test/repo/blaxel-worker-lease.test.ts @@ -0,0 +1,38 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { workerLeaseCommand } from "../../scripts/test/blaxel-worker-lease.mjs"; + +// Real generated lock command, Linux flock/procfs and disposable files. The +// fixture-token filter substitutes only the whole-worker audit's scope. +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +function fixture() { + const tmpRoot = mkdtempSync(path.join(os.tmpdir(), "lease-seam-")); roots.push(tmpRoot); + const input = { lockId: "fixture-lock", runId: "fixture-run", shardIndex: 1, shardTotal: 1, attemptId: "fixture-attempt", owner: "fixture", expiresAt: Math.floor(Date.now() / 1000) + 1200, tmpRoot, resultsRoot: path.join(tmpRoot, "results"), auditTokens: ["fixture-lock"] }; + const recordPath = path.join(tmpRoot, "tv-testshard.lock", "owner.json"); + const run = (extra = {}) => spawnSync("bash", ["-c", workerLeaseCommand({ ...input, ...extra })], { encoding: "utf8", env: process.env, timeout: 15_000 }); + return { input, recordPath, run, record: () => JSON.parse(readFileSync(recordPath, "utf8")) }; +} + +test.skipIf(process.platform !== "linux")("acquisition writes a renewable deadline and protects the lease from timestamp-only runners", () => { + const f = fixture(); + const before = Date.now(); + const acquired = f.run(); + expect(acquired.status, acquired.stderr).toBe(0); + expect(f.record()).toMatchObject({ lockId: f.input.lockId, expiresAt: 0, attemptId: f.input.attemptId }); + expect(f.record().lastRenewal).toBeGreaterThanOrEqual(before); + expect(f.record().recoverAfter - f.record().lastRenewal).toBe(1_200_000); +}); + +test.skipIf(process.platform !== "linux")("a quiet expired renewable lease recovers through cleanup before replacement", () => { + const f = fixture(); + mkdirSync(path.dirname(f.recordPath)); + writeFileSync(f.recordPath, JSON.stringify({ lockId: "abandoned", runId: "old-run", shardIndex: 1, attemptId: "old-attempt", expiresAt: 0, lastRenewal: 1, recoverAfter: 2, dispatch: null })); + const acquired = f.run(); + expect(acquired.status, acquired.stderr).toBe(0); + expect(f.record().lockId).toBe(f.input.lockId); + expect(f.record().recovery).toMatchObject({ previous: { lockId: "abandoned", runId: "old-run" }, cleanup: "confirmed", discardedReports: true }); +}); From 2c0f5d6b1a86df66d2d242cc23fbefa07eb0610a Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:50:38 +0000 Subject: [PATCH 35/64] Implement guarded worker ownership and result retention --- ...arness-resource-cleanup-review-response.md | 20 ++ scripts/manage-blaxel-testshards.mjs | 24 +- scripts/run-blaxel-testshards.mjs | 338 +++++++++--------- scripts/test/blaxel-lease-lifecycle.d.mts | 2 + scripts/test/blaxel-lease-lifecycle.mjs | 29 ++ scripts/test/blaxel-pool-lifecycle.d.mts | 3 + scripts/test/blaxel-pool-lifecycle.mjs | 14 + scripts/test/blaxel-pool.mjs | 2 +- scripts/test/blaxel-worker-lease.d.mts | 11 + scripts/test/blaxel-worker-lease.mjs | 272 +++++++++++++- scripts/test/cli.mjs | 1 + scripts/test/resource-diagnostics.mjs | 3 +- scripts/test/surface-supervisor.mjs | 2 +- test/helpers/owned-process.ts | 6 +- test/repo/blaxel-lease-contracts.test.ts | 116 ++++++ test/repo/blaxel-worker-lease.test.ts | 14 + 16 files changed, 652 insertions(+), 205 deletions(-) create mode 100644 scripts/test/blaxel-lease-lifecycle.d.mts create mode 100644 scripts/test/blaxel-lease-lifecycle.mjs create mode 100644 scripts/test/blaxel-pool-lifecycle.d.mts create mode 100644 scripts/test/blaxel-pool-lifecycle.mjs create mode 100644 scripts/test/blaxel-worker-lease.d.mts create mode 100644 test/repo/blaxel-lease-contracts.test.ts diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 05442aee..01e81fbb 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -240,3 +240,23 @@ Slice 3 starts with real generated lock-command checks on disposable Linux roots The first checkpoint only extracts the existing command without changing its timestamp-only behavior; the new assertions require renewable ownership and confirmed quiet-orphan recovery. Execution is pending on the private Blaxel pool. + +## Slice 3 first red and Primitive observation + +At `1320c957`, private canonical run +`.test-runs/2026-10-27T18-41-03-525Z-p19298-r7dc71e154247c385` +reached both native lock assertions and failed them with completed infrastructure: +the timestamp-only record lacks the renewable identity and cannot recover an +expired renewable lease. Defaults were unchanged: no unit retries, two allowed +infrastructure retries (none used). + +Primitive flagged `dec_d81a353e` during pool-lifecycle edits. The ordinary CLI was +absent; npm returned ETARGET and its installed launcher failed with a dynamic +require error. A temporary loader recovered the decision text: it repeats the +earlier lock directive, including the literal no-expiry language, without the +later reviewed automatic-recovery clarification. Recorded the check as followed: +normal runs retain ownership through download; expired renewal authorizes owned +process cleanup, never reuse beneath live work. The converged specs govern that +clarification. The supervisor explicitly directed no further Primitive debugging. +Its partial/truncated decision scan is not a complete authority check. No tests, +builds or application processes ran on the shared host. diff --git a/scripts/manage-blaxel-testshards.mjs b/scripts/manage-blaxel-testshards.mjs index 58996712..7c2c348b 100644 --- a/scripts/manage-blaxel-testshards.mjs +++ b/scripts/manage-blaxel-testshards.mjs @@ -2,6 +2,9 @@ import { SandboxInstance } from "@blaxel/core"; import { readFileSync } from "node:fs"; import process from "node:process"; +import { randomUUID } from "node:crypto"; +import { workerLeaseCommand } from "./test/blaxel-worker-lease.mjs"; +import { publishWorker, deleteWorker } from "./test/blaxel-pool-lifecycle.mjs"; import { listBlaxelPoolSandboxes } from "./test/blaxel-pool.mjs"; import { DEFAULT_BLAXEL_POOL_SIZE } from "./testshard-constants.mjs"; @@ -56,14 +59,12 @@ if (command === "ensure") { region, labels: { project: "television", - purpose: "testshards", + purpose: "testshards-provisioning", pool, arch, }, }); - await sandbox.wait(); - await provisionSandboxRuntime(sandbox); - await verifySandboxArchitecture(sandbox); + await publishWorker(sandbox, { provision: provisionSandboxRuntime, verify: verifySandboxArchitecture, update: SandboxInstance.updateMetadata }); console.log(` ${sandbox.metadata?.name}: ${sandbox.status}`); } process.exit(0); @@ -71,11 +72,20 @@ if (command === "ensure") { if (command === "delete") { const sandboxes = await listPoolSandboxes(); + let refused = false; for (const sandbox of sandboxes) { - console.log(`deleting ${sandbox.metadata?.name}`); - await SandboxInstance.delete(sandbox.metadata.name); + try { + await deleteWorker(sandbox, { reserve: reserveDeletion, update: SandboxInstance.updateMetadata, remove: SandboxInstance.delete }); + console.log(`deleted ${sandbox.metadata.name}`); + } catch (error) { refused = true; console.error(`held ${sandbox.metadata.name}: ${error.message}`); } } - process.exit(0); + process.exit(refused ? 1 : 0); +} + +async function reserveDeletion(sandbox) { + const script = ["set -eo pipefail", 'export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"', '. "$NVM_DIR/nvm.sh"', "nvm use --silent default", "set -u", workerLeaseCommand({ operation: "delete", lockId: `delete-${randomUUID()}` })].join("; "); + const result = await sandbox.process.exec({ name: `tv-delete-reservation-${randomUUID()}`, command: `bash -lc ${shellQuote(script)}`, waitForCompletion: true, timeout: 30 }); + if (result.exitCode !== 0) throw new Error(`ownership check refused deletion: ${result.logs ?? result.exitCode}`); } async function provisionSandboxRuntime(sandbox) { diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index 7d0c4621..8eaadd97 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -14,7 +14,9 @@ import { RESOURCE_SIDECAR_BYTES, RESOURCE_ATTEMPT_BYTES } from "./test/resource- import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; -import { workerLeaseCommand } from "./test/blaxel-worker-lease.mjs"; +import { randomUUID } from "node:crypto"; +import { renewLease, finishLease } from "./test/blaxel-lease-lifecycle.mjs"; +import { workerLeaseCommand, workerEntryCommand, shardScratchPath } from "./test/blaxel-worker-lease.mjs"; import path from "node:path"; import process from "node:process"; @@ -145,15 +147,17 @@ async function runStaleOwnerAudit() { lease.processName = processName; try { console.log(`acquired stale-owner audit lease: ${lease.sandbox.metadata.name}`); - const setup = await runSystemDependencySetup(lease); - lease.processName = processName; - console.log(`read-only diagnostic command: ${[command, ...args].map(shellQuote).join(" ")}`); - const proc = await lease.sandbox.process.exec({ - name: processName, - command: [command, ...args].map(shellQuote).join(" "), - waitForCompletion: true, - timeout: 30, - }); + const remoteScript = `${lease.input.scratch}/audit.sh`; + lease.pendingWrites = lease.sandbox.fs.write(remoteScript, `#!/usr/bin/env bash\nset -e\n${systemDependenciesCommand()}\n${[command, ...args].map(shellQuote).join(" ")}\n`); + await lease.pendingWrites; + await leaseOperation(lease, "prepare"); + await leaseOperation(lease, "dispatch", { processName }); + lease.executionRequested = true; + const setup = { name: "system-deps" }; + const proc = await (lease.dispatchRequest = lease.sandbox.process.exec({ + name: processName, command: workerEntryCommand({ ...lease.input, processName }, remoteScript), + waitForCompletion: true, timeout: timeouts.workerTimeoutSeconds, + })); const rawLogs = await lease.sandbox.process.logs(processName, "all").catch(() => proc.logs ?? ""); const logs = chunkToText(rawLogs); if (proc.exitCode !== 0) throw new Error(`read-only stale-owner diagnostic exited ${proc.exitCode}: ${logs.trim()}`); @@ -172,6 +176,7 @@ async function runStaleOwnerAudit() { ...audit, }; console.log(`${JSON.stringify(result, null, 2)}`); + lease.retrieved = true; if (!audit.capabilities.procfs.readable || audit.processScan.status !== "ok") return 2; if (!audit.capabilities.ss.available || !audit.capabilities.ss.usable) return 2; if (staleOwnerCount > 0) return 3; @@ -183,24 +188,6 @@ async function runStaleOwnerAudit() { } } -async function runSystemDependencySetup(lease) { - const args = ["-lc", systemDependenciesCommand()]; - const processName = `tv-system-deps-${runId}`.replace(/[^a-zA-Z0-9-]/g, "-"); - lease.processName = processName; - console.log(`system dependency setup: ${["bash", ...args].map(shellQuote).join(" ")}`); - const proc = await lease.sandbox.process.exec({ - name: processName, - command: ["bash", ...args].map(shellQuote).join(" "), - waitForCompletion: true, - timeout: timeouts.workerTimeoutSeconds, - }); - const rawLogs = await lease.sandbox.process.logs(processName, "all").catch(() => proc.logs ?? ""); - const logs = chunkToText(rawLogs); - if (logs.trim()) console.log(logs.trimEnd()); - if (proc.exitCode !== 0) throw new Error(`Blaxel system dependency setup exited ${proc.exitCode}: ${logs.trim()}`); - return { name: "system-deps", command: ["bash", ...args], exitCode: proc.exitCode }; -} - function validateStaleOwnerAuditOptions(parsed) { const allowed = new Set(["audit-stale-owners", "pool", "arch"]); const unexpected = Object.keys(parsed).filter((key) => !allowed.has(key)); @@ -240,7 +227,7 @@ async function runAttempt({ attempt, shardIndices, excludedSandboxes = [], outpu maxConcurrency: MAX_PARALLEL_LEASE_ACQUISITIONS, acquire: async ({ shardIndex, slot }) => { attemptState.shardStartedAt.set(shardIndex, Date.now()); - const lease = await acquireAny(sandboxes, { shardIndex, slot, requestedWorkers: shardIndices.length }); + const lease = await acquireAny(sandboxes, { shardIndex, slot, requestedWorkers: shardIndices.length, attempt: attempt + 1 }); if (lease) attemptState.leases.set(shardIndex, lease); return lease; }, @@ -267,12 +254,12 @@ async function runAttempt({ attempt, shardIndices, excludedSandboxes = [], outpu console.log(`acquired: ${formatShardLeases(acquired)}`); console.log(`starting: ${formatShardLeases(acquired)}`); results = await Promise.all(acquired.map(async (lease) => { - const result = await runShard(lease, outputDir, { + const result = await (lease.workPromise = runShard(lease, outputDir, { attempt, onExecutionRequested: () => attemptState.executionRequested.add(lease.shardIndex), onExecutionStarted: () => attemptState.executionStarted.add(lease.shardIndex), onResultProgress: (progress) => attemptState.partialResults.set(lease.shardIndex, progress), - }); + })); attemptState.partialResults.delete(lease.shardIndex); attemptState.completedResults.set(lease.shardIndex, result); return result; @@ -299,6 +286,7 @@ async function runAttempt({ attempt, shardIndices, excludedSandboxes = [], outpu durationMs: Date.now() - startedAt, shards: results.sort((a, b) => a.shardIndex - b.shardIndex), }; + if (shuttingDown) await shutdownPromise; writeProviderReports(outputDir, report); return report; } finally { @@ -334,29 +322,42 @@ function hashString(value) { return hash >>> 0; } -async function acquireAny(sandboxes, { shardIndex, slot, requestedWorkers, verifyArchitecture = true }) { - const lockId = `${runId}-shard-${shardIndex}`; - const expiresAt = Math.floor(Date.now() / 1000) + timeouts.lockTtlSeconds; +async function leaseOperation(lease, operation, extra = {}) { + if (lease.lost) throw lease.lost; + const body = ["set -eo pipefail", 'export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"', '. "$NVM_DIR/nvm.sh"', "nvm use --silent default", "set -u", workerLeaseCommand({ ...lease.input, operation, ...extra })].join("; "); + const result = await lease.sandbox.process.exec({ name: `tv-${operation}-${randomUUID()}`, command: `bash -lc ${shellQuote(body)}`, waitForCompletion: true, timeout: 30 }); + const lines = chunkToText(result.logs ?? "").trim().split("\n"); + let response; + try { response = JSON.parse(lines.at(-1)); } catch { throw new Error(`worker ${operation} returned no ownership evidence (exit ${result.exitCode})`); } + if (result.exitCode !== 0 || !response.ok) throw new Error(`worker ${operation}: ${response.reason ?? "failed"}`); + return response; +} + +async function acquireAny(sandboxes, { shardIndex, slot, requestedWorkers, verifyArchitecture = true, attempt = 1 }) { + const lockId = `${runId}-s${shardIndex}-a${attempt}-${randomUUID().slice(0, 8)}`; + const input = { lockId, runId, shardIndex, attempt, attemptId: lockId, owner: `${os.hostname()}:${process.pid}`, ttlSeconds: timeouts.lockTtlSeconds, audit: auditStaleOwners }; + input.scratch = shardScratchPath(input); for (const sandbox of acquisitionOrder(sandboxes, { slot, requestedWorkers })) { - if (sandbox.__leased) continue; - const command = `bash -lc ${shellQuote(workerLeaseCommand({ lockId, runId, shardIndex, shardTotal, owner: `${os.hostname()}:${process.pid}`, expiresAt }))}`; - const proc = await sandbox.process.exec({ name: `tv-lock-${lockId}`, command, waitForCompletion: true, timeout: 30 }); - if (proc.exitCode === 0) { - let retainLock = false; - try { - if (verifyArchitecture && !(await verifySandboxArchitecture(sandbox))) { - sandbox.__leased = true; - continue; - } - sandbox.__leased = true; - const lease = { sandbox, lockId, shardIndex, processName: null }; - activeLeases.set(lockId, lease); - retainLock = true; - return lease; - } finally { - if (!retainLock) await releaseLock(sandbox, lockId); - } + if (sandbox.__leased || shuttingDown) continue; + const lease = { sandbox, lockId, shardIndex, input: { ...input, worker: sandbox.metadata.name }, processName: null }; + try { + await leaseOperation(lease, "acquire"); + } catch (error) { + console.error(`Skipping ${sandbox.metadata.name}: ${error.message}`); + continue; + } + sandbox.__leased = true; + activeLeases.set(lockId, lease); + lease.stopRenewal = renewLease({ + intervalMs: Math.max(250, Math.min(30_000, timeouts.lockTtlSeconds * 1000 / 4)), + renew: () => leaseOperation(lease, "renew"), + onLost: (error) => { lease.lost = error; console.error(`Lease renewal lost on ${sandbox.metadata.name}: ${error.message}`); }, + }); + if (verifyArchitecture && !(await verifySandboxArchitecture(sandbox))) { + await stopAndReleaseLease(lease, { killProcess: false }); + continue; } + return lease; } return null; } @@ -399,7 +400,7 @@ function acquisitionOrder(sandboxes, { slot, requestedWorkers }) { async function runShard(lease, outputDir, { attempt, onExecutionRequested = () => {}, onExecutionStarted = () => {}, onResultProgress = () => {} } = {}) { const { sandbox, shardIndex, lockId } = lease; - const diagnosticAttemptId = `${lockId}-attempt-${attempt + 1}`; + const diagnosticAttemptId = lease.input.attemptId; const startedAt = Date.now(); if (lease.restorePhase) { lease.restorePhase.completedAt = new Date(startedAt).toISOString(); @@ -408,52 +409,74 @@ async function runShard(lease, outputDir, { attempt, onExecutionRequested = () = const overallStarted = lease.restorePhase ? Date.parse(lease.restorePhase.startedAt) : startedAt; const processName = `tv-testshard-${runId}-s${shardIndex}`.replace(/[^a-zA-Z0-9-]/g, "-"); lease.processName = processName; - const remoteScript = `/tmp/${processName}.sh`; + const remoteScript = `${lease.input.scratch}/worker.sh`; + lease.remoteScript = remoteScript; const logPath = path.join(outputDir, `shard-${shardIndex}.log`); lease.logPath = logPath; const summaryPath = path.join(outputDir, `shard-${shardIndex}.json`); - const planRemotePath = shardPlanRaw ? `/tmp/${processName}.plan.json` : null; - const dispatch = await transferShardInputs({ + const planRemotePath = shardPlanRaw ? `${lease.input.scratch}/plan.json` : null; + const dispatch = await (lease.pendingWrites = transferShardInputs({ write: (remotePath, contents) => sandbox.fs.write(remotePath, contents), planRemotePath, planRaw: shardPlanRaw, remoteScript, - remoteScriptContents: shardScript({ shardIndex, lockId, planRemotePath, diagnosticAttemptId }), - }); + remoteScriptContents: shardScript({ shardIndex, lockId, planRemotePath, diagnosticAttemptId, scratch: lease.input.scratch }), + })); if (!dispatch.ok) { appendLog(logPath, `\n[coordinator] ${dispatch.failureMessage}\n`); return dispatchFailureResult({ lease, outputDir, startedAt, logPath, dispatch }); } - const command = `bash ${remoteScript}`; + const command = workerEntryCommand({ ...lease.input, processName }, remoteScript); let stream; let retainedResultProgress = null; let resourceDiagnosticsPaths = []; const copiedArtifacts = new Map(); let failureStep = "worker-execution"; + let exitCode = null; + let executionError = null; try { + await leaseOperation(lease, "prepare"); + await leaseOperation(lease, "dispatch", { processName }); lease.executionRequested = true; onExecutionRequested(); - const proc = await sandbox.process.exec({ - name: processName, - command, - env: { GH_TOKEN: blaxelGithubToken }, - timeout: timeouts.workerTimeoutSeconds, - keepAlive: true, - }); + const proc = await (lease.dispatchRequest = sandbox.process.exec({ name: processName, command, env: { GH_TOKEN: blaxelGithubToken }, timeout: timeouts.workerTimeoutSeconds, keepAlive: true })); onExecutionStarted(); stream = sandbox.process.streamLogs(processName, { - onStdout: (chunk) => appendLog(logPath, chunk), - onStderr: (chunk) => appendLog(logPath, chunk), + onStdout: (chunk) => appendLog(logPath, chunk), onStderr: (chunk) => appendLog(logPath, chunk), onError: (error) => console.error(`Log stream failed for shard ${shardIndex}: ${error.message}`), }); const completed = await sandbox.process.wait(processName, { maxWait: timeouts.workerTimeoutSeconds * 1000, interval: 5000 }); lease.executionCompletionConfirmed = true; - stream?.close?.(); - const logs = await sandbox.process.logs(processName, "all").catch(() => ""); - if (logs && !fs.existsSync(logPath)) fs.writeFileSync(logPath, logs); - failureStep = "report-download"; + exitCode = completed.exitCode ?? proc.exitCode ?? null; + } catch (error) { executionError = error; } + stream?.close?.(); + let result; + try { + await finishLease({ + settleWrites: async () => { await lease.pendingWrites; }, + assertOwned: () => { if (lease.lost) throw lease.lost; }, + teardown: () => confirmTeardown(lease), + download: async () => { + await beforeDownload(lease); + const logs = await sandbox.process.logs(processName, "all"); + appendLog(logPath, `\n[coordinator] final worker log:\n${chunkToText(logs)}`); + failureStep = "report-download"; + result = await downloadResult(); + lease.retrieved = true; + }, + release: async () => { await leaseOperation(lease, "release", { retrieved: true }); lease.released = true; }, + }); + if (executionError) return { ...result, status: "infra-failed", ...classifyBlaxelCoordinatorFailure({ failureStep: "worker-execution", message: executionError.message }), failureMessage: executionError.message }; + return result; + } catch (error) { + appendLog(logPath, `\n[coordinator] ${error.stack || error.message}\n`); + return { ...retainedResultProgress, shardIndex, shardTotal, sandbox: sandbox.metadata.name, outputDir, status: "infra-failed", exitCode, startedAt: new Date(overallStarted).toISOString(), completedAt: new Date().toISOString(), durationMs: Date.now() - overallStarted, logPath, resourceDiagnosticsPaths, tasks: retainedResultProgress?.tasks ?? [], ...classifyBlaxelCoordinatorFailure({ failureStep, message: error.message }), failureMessage: error.message }; + } finally { await lease.stopRenewal?.(); } + + async function downloadResult() { const reportStarted = Date.now(); - const exitCode = completed.exitCode ?? proc.exitCode ?? null; + await leaseOperation(lease, "check"); + await retainAttemptEvidence(lease, outputDir); const plannedShard = shardPlan?.shards?.[shardIndex - 1]; let shardSummary = null; let shardSummaryError = null; @@ -502,8 +525,7 @@ async function runShard(lease, outputDir, { attempt, onExecutionRequested = () = appendLog(logPath, `\n[coordinator] ${shardSummaryError}\n`); } } catch (error) { - shardSummaryError = `Failed to read shard summary: ${error.message}`; - appendLog(logPath, `\n[coordinator] ${shardSummaryError}\n`); + throw new Error(`Failed to read shard summary: ${error.message}`); } retainResultProgress(); const remoteShardNativePath = `/workspace/television/.testshards/results/native-shard-${shardIndex}.json`; @@ -543,9 +565,7 @@ async function runShard(lease, outputDir, { attempt, onExecutionRequested = () = const setupRaw = await sandbox.fs.read(`/workspace/television/.testshards/results/setup-shard-${shardIndex}.json`); fs.writeFileSync(path.join(outputDir, `setup-shard-${shardIndex}.json`), setupRaw.endsWith("\n") ? setupRaw : `${setupRaw}\n`); setupTimings = [lease.restorePhase, ...JSON.parse(setupRaw)].filter(Boolean); - } catch { - setupTimings = [lease.restorePhase].filter(Boolean); - } + } catch (error) { throw new Error(`Failed to retain setup timings: ${error.message}`); } retainResultProgress(); const identityError = shardPlan && shardSummary ? validateDownloadedShardSummary(shardSummary, shardPlan, shardIndex) : null; const resultError = downloadedResultError(shardSummary, exitCode, shardIndex); @@ -553,6 +573,7 @@ async function runShard(lease, outputDir, { attempt, onExecutionRequested = () = shardSummaryError = identityError ?? resultError; appendLog(logPath, `\n[coordinator] ${shardSummaryError}\n`); } + if (!shardSummary || identityError || resultError) throw new Error(shardSummaryError ?? "missing shard summary"); const hasSummaryFile = fs.existsSync(summaryPath); const status = classifyDownloadedShardStatus({ identityError, shardSummary, exitCode, hasSummaryFile, setupTimings }); const completedAt = new Date().toISOString(); @@ -563,39 +584,7 @@ async function runShard(lease, outputDir, { attempt, onExecutionRequested = () = failureStep: hasSummaryFile ? "summary-parse" : "summary-read", failureMessage: shardSummaryError ?? `Shard ${shardIndex}/${shardTotal} produced no readable summary.`, } : null; - return { shardIndex, shardTotal, shard: `${shardIndex}/${shardTotal}`, sandbox: sandbox.metadata.name, outputDir, status, exitCode, startedAt: new Date(overallStarted).toISOString(), completedAt, durationMs: Date.now() - overallStarted, summaryPath: fs.existsSync(summaryPath) ? summaryPath : null, summaryError: shardSummaryError, nativeResultPath, resourceDiagnosticsPaths, logPath, setupTimings, planId: shardSummary?.planId ?? shardPlan?.planId, timingProvider: shardSummary?.timingProvider ?? shardPlan?.timingProvider, testedTree: shardSummary?.testedTree ?? shardPlan?.testedTree, testedCommit: shardSummary?.testedCommit ?? shardPlan?.testedCommit, predictedDurationMs: shardSummary?.predictedDurationMs, phases: [...(shardSummary?.phases ?? []), reportPhase], assignedFiles: shardSummary?.assignedFiles ?? [], collectedFiles: shardSummary?.collectedFiles ?? [], failureKind: identityError ? "plan-validation" : resultError ? "result-report" : missingSummaryFailure?.failureKind ?? shardSummary?.failureKind ?? failedInfrastructureTask?.failureKind, failureStep: identityError ? "plan-validation" : resultError ? "summary-validation" : missingSummaryFailure?.failureStep ?? shardSummary?.failureStep ?? failedInfrastructureTask?.failureStep, failureMessage: identityError ?? resultError ?? missingSummaryFailure?.failureMessage ?? shardSummary?.failureMessage ?? failedInfrastructureTask?.failureMessage, processLeaks: shardSummary?.processLeaks ?? [], tasks }; - } catch (error) { - stream?.close?.(); - resourceDiagnosticsPaths = await copyResourceDiagnostics(sandbox, { outputDir, shardIndex, diagnosticAttemptId, copiedArtifacts }); - appendLog(logPath, `\n[coordinator] ${error.stack || error.message}\n`); - const completedAt = new Date().toISOString(); - const plannedShard = shardPlan?.shards?.[shardIndex - 1]; - const failure = classifyBlaxelCoordinatorFailure({ failureStep, message: error.message }); - return { - ...retainedResultProgress, - resourceDiagnosticsPaths, - shardIndex, - shardTotal, - shard: `${shardIndex}/${shardTotal}`, - sandbox: sandbox.metadata.name, - outputDir, - status: "infra-failed", - error: error.message, - ...failure, - failureMessage: error.message, - planId: retainedResultProgress?.planId ?? shardPlan?.planId, - timingProvider: retainedResultProgress?.timingProvider ?? shardPlan?.timingProvider, - testedCommit: retainedResultProgress?.testedCommit ?? shardPlan?.testedCommit, - testedTree: retainedResultProgress?.testedTree ?? shardPlan?.testedTree, - predictedDurationMs: retainedResultProgress?.predictedDurationMs ?? plannedShard?.predictedDurationMs, - startedAt: retainedResultProgress?.startedAt ?? new Date(overallStarted).toISOString(), - completedAt, - durationMs: Date.now() - overallStarted, - logPath, - assignedFiles: retainedResultProgress?.assignedFiles ?? plannedShard?.surfaces?.flatMap((surface) => surface.files.map((file) => file.path)).sort() ?? [], - collectedFiles: retainedResultProgress?.collectedFiles ?? [], - tasks: retainedResultProgress?.tasks ?? [], - }; + return { shardIndex, shardTotal, shard: `${shardIndex}/${shardTotal}`, sandbox: sandbox.metadata.name, outputDir, status, exitCode, startedAt: new Date(overallStarted).toISOString(), completedAt, durationMs: Date.now() - overallStarted, summaryPath: fs.existsSync(summaryPath) ? summaryPath : null, summaryError: shardSummaryError, nativeResultPath, resourceDiagnosticsPaths, remoteScript, scratch: lease.input.scratch, logPath, setupTimings, planId: shardSummary?.planId ?? shardPlan?.planId, timingProvider: shardSummary?.timingProvider ?? shardPlan?.timingProvider, testedTree: shardSummary?.testedTree ?? shardPlan?.testedTree, testedCommit: shardSummary?.testedCommit ?? shardPlan?.testedCommit, predictedDurationMs: shardSummary?.predictedDurationMs, phases: [...(shardSummary?.phases ?? []), reportPhase], assignedFiles: shardSummary?.assignedFiles ?? [], collectedFiles: shardSummary?.collectedFiles ?? [], failureKind: identityError ? "plan-validation" : resultError ? "result-report" : missingSummaryFailure?.failureKind ?? shardSummary?.failureKind ?? failedInfrastructureTask?.failureKind, failureStep: identityError ? "plan-validation" : resultError ? "summary-validation" : missingSummaryFailure?.failureStep ?? shardSummary?.failureStep ?? failedInfrastructureTask?.failureStep, failureMessage: identityError ?? resultError ?? missingSummaryFailure?.failureMessage ?? shardSummary?.failureMessage ?? failedInfrastructureTask?.failureMessage, processLeaks: shardSummary?.processLeaks ?? [], tasks }; } } @@ -793,14 +782,11 @@ function systemDependenciesCommand() { return 'if ! command -v python3 >/dev/null 2>&1 || ! command -v xauth >/dev/null 2>&1 || ! command -v ss >/dev/null 2>&1 || ! ldconfig -p 2>/dev/null | grep -q "libgtk-3.so.0" || ! ldconfig -p 2>/dev/null | grep -q "libXtst.so.6"; then apt-get update && apt-get install -y --no-install-recommends python3 xauth libgtk-3-0 libxtst6 iproute2; fi'; } -function shardScript({ shardIndex, lockId, planRemotePath, diagnosticAttemptId }) { +function shardScript({ shardIndex, lockId, planRemotePath, diagnosticAttemptId, scratch }) { return `#!/usr/bin/env bash set -euo pipefail -lock=/tmp/tv-testshard.lock shard_plan=${shellQuote(planRemotePath ?? "")} -release_lock() { - if [ -f "$lock/owner.json" ] && grep -q '"lockId":"${lockId}"' "$lock/owner.json"; then rm -rf "$lock"; fi -} +export TMPDIR=${shellQuote(`${scratch}/tmp`)} export CI=1 export SKIP_ACP_TESTS=1 export NVM_DIR="\${NVM_DIR:-$HOME/.nvm}" @@ -815,9 +801,9 @@ export TV_TEST_SHARD_INDEX=${shardIndex} export TV_TEST_SHARD_ATTEMPT=${shellQuote(diagnosticAttemptId)} mkdir -p /workspace /cache/npm if [ -z "\${GH_TOKEN:-}" ]; then echo "GH_TOKEN missing" >&2; exit 2; fi -setup_metrics=/tmp/tv-testshard-setup-${shardIndex}.jsonl -deps_cache_status=/tmp/tv-testshard-deps-cache-${shardIndex}.txt -owner_report=/tmp/tv-stale-owner-${shardIndex}.json +setup_metrics=${shellQuote(`${scratch}/setup.jsonl`)} +deps_cache_status=${shellQuote(`${scratch}/deps.txt`)} +owner_report=${shellQuote(`${scratch}/owners.json`)} : > "$setup_metrics" : > "$deps_cache_status" export TV_TEST_DEPS_CACHE_STATUS_FILE="$deps_cache_status" @@ -863,6 +849,8 @@ const [input, setupOutput, shardOutput, depsCacheStatusFile] = process.argv.slic const depsCacheStatus = fs.existsSync(depsCacheStatusFile) ? fs.readFileSync(depsCacheStatusFile, 'utf8').trim() : ''; const rows = (fs.existsSync(input) ? fs.readFileSync(input, 'utf8').split(/\\n/).filter(Boolean).map((line) => JSON.parse(line)) : []) .map((row) => row.name === 'deps' && ['hit', 'miss'].includes(depsCacheStatus) ? { ...row, cacheStatus: depsCacheStatus } : row); +const cleanupPath = '/workspace/television/.testshards/results/temporary-cleanup.json'; +if (fs.existsSync(cleanupPath)) rows.unshift({ name: 'temporary-cleanup', status: 0, durationMs: 0, cleanup: JSON.parse(fs.readFileSync(cleanupPath, 'utf8')) }); fs.writeFileSync(setupOutput, JSON.stringify(rows, null, 2) + "\\n"); if (!fs.existsSync(shardOutput)) { const failed = rows.find((row) => row.status && row.status !== 0); @@ -887,17 +875,16 @@ if (!fs.existsSync(shardOutput)) { } METRICS_NODE } -trap 'write_setup_metrics; rm -f "$owner_report" "$deps_cache_status"; if [ -n "$shard_plan" ]; then rm -f "$shard_plan"; fi; release_lock' EXIT -rm -rf /workspace/television/.testshards 2>/dev/null || true -cat > /tmp/tv-git-askpass.sh <<'ASKPASS' +trap 'write_setup_metrics' EXIT +cat > ${scratch}/git-askpass.sh <<'ASKPASS' #!/usr/bin/env bash case "$1" in *Username*) echo x-access-token ;; *) echo "$GH_TOKEN" ;; esac ASKPASS -chmod 700 /tmp/tv-git-askpass.sh -export GIT_ASKPASS=/tmp/tv-git-askpass.sh +chmod 700 ${scratch}/git-askpass.sh +export GIT_ASKPASS=${scratch}/git-askpass.sh export GIT_TERMINAL_PROMPT=0 run_step checkout bash -lc ${shellQuote(blaxelCheckoutScript({ repoUrl, commit }))} cd /workspace/television @@ -964,7 +951,7 @@ elif [ ${shellQuote(suite)} = target ]; then export TV_TARGET_RUN_ID=${shellQuote(`${runId}:${shardIndex}`)} export TV_TARGET_NAME=${shellQuote(`${targetOptions()?.workspace ?? "target"} ${targetOptions()?.file ?? ""} ${targetOptions()?.grep ?? ""}`)} export TV_TARGET_RUNNER=${shellQuote(targetOptions()?.runner ?? "target")} - export TV_TARGET_CONFIG=${shellQuote(targetOptions()?.config ? path.posix.join("/workspace/television", targetOptions().config) : "")} + export TV_TARGET_CONFIG=${shellQuote(targetOptions()?.config ? path.posix.join("/workspace/television", targetOptions().cwd, targetOptions().config) : "")} export TV_TARGET_COMMAND=${shellQuote(targetCommand())} export TV_TARGET_LIFECYCLE_RESULT="/workspace/television/.testshards/results/target-lifecycle-${shardIndex}.json" export TV_TARGET_NATIVE_RESULT="/workspace/television/.testshards/results/native-shard-${shardIndex}.json" @@ -1030,63 +1017,57 @@ exit "$test_status" `; } -async function releaseLock(sandbox, lockId) { - const command = `bash -lc ${shellQuote(`lock=/tmp/tv-testshard.lock; if [ -f "$lock/owner.json" ] && grep -q '\"lockId\":\"${lockId}\"' "$lock/owner.json"; then rm -rf "$lock"; fi`)}`; - await sandbox.process.exec({ name: `tv-release-${lockId}`, command, waitForCompletion: true, timeout: 30 }).catch(() => {}); +async function confirmTeardown(lease) { + if (lease.teardownConfirmed) return; + const result = await leaseOperation(lease, "teardown"); + lease.teardownConfirmed = true; + if (lease.logPath) appendLog(lease.logPath, `\n[coordinator] owner teardown: ${JSON.stringify(result.record.teardown)}\n`); } -async function stopAndReleaseLease(lease, { killProcess }) { - if (killProcess && lease.processName) { - await lease.sandbox.process.kill(lease.processName).catch(() => {}); - try { - await reapUnconfirmedShardOwners(lease); - } catch (error) { - if (lease.logPath) appendLog(lease.logPath, `\n[coordinator] Owner cleanup was not confirmed; retaining lease ${lease.lockId} until TTL: ${error.stack || error.message}\n`); - throw error; - } +async function retainAttemptEvidence(lease, directory) { + if (!lease.executionRequested) { lease.retrieved = true; return; } + const root = "/workspace/television/.testshards/results"; + const identity = JSON.parse(await lease.sandbox.fs.read(`${root}/attempt.json`)); + for (const key of ["lockId", "runId", "shardIndex", "attemptId"]) if (identity[key] !== lease.input[key]) throw new Error(`attempt evidence identity mismatch: ${key}`); + const evidence = JSON.parse(await lease.sandbox.fs.read(`${root}/evidence.json`)); + if (evidence.runId !== lease.input.runId || evidence.attemptId !== lease.input.attemptId || !Array.isArray(evidence.files)) throw new Error("invalid retained-evidence inventory"); + for (const name of evidence.files) { + if (typeof name !== "string" || name.startsWith("/") || name.split("/").includes("..")) throw new Error("invalid evidence path"); + const raw = await lease.sandbox.fs.read(`${root}/${name}`); + if (name.endsWith(".json")) JSON.parse(raw); + const target = path.join(directory, "artifacts", `shard-${lease.shardIndex}`, name); + fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, raw); } - await releaseLock(lease.sandbox, lease.lockId); } -async function reapUnconfirmedShardOwners(lease) { - const processName = `tv-owner-cleanup-${lease.lockId}`.replace(/[^a-zA-Z0-9-]/g, "-"); - const reportPath = `/tmp/${processName}.json`; - const script = [ - "set -eo pipefail", - 'export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"', - '. "$NVM_DIR/nvm.sh"', - "nvm use --silent default", - "set -u", - "cd /workspace/television", - "status=0", - `node scripts/test/stale-owner-reaper-cli.mjs --output ${shellQuote(reportPath)} || status=$?`, - `cat ${shellQuote(reportPath)}`, - 'exit "$status"', - ].join("; "); - const completed = await lease.sandbox.process.exec({ - name: processName, - command: `bash -lc ${shellQuote(script)}`, - waitForCompletion: true, - timeout: 30, - }); - const raw = await lease.sandbox.fs.read(reportPath); - let report; - try { - report = JSON.parse(raw); - } catch (error) { - throw new Error(`Unconfirmed shard owner cleanup returned invalid JSON: ${error.message}`); - } - const validExit = completed.exitCode === 0 || completed.exitCode === 1; - const consistentExit = completed.exitCode === (report.staleDetected ? 1 : 0); - const cleanupConfirmed = report.cleanupConfirmed === true && - Array.isArray(report.liveOwners) && report.liveOwners.length === 0 && - Array.isArray(report.remainingStaleOwners) && report.remainingStaleOwners.length === 0 && - Array.isArray(report.processLeaks); - if (!validExit || !consistentExit || !cleanupConfirmed) { - throw new Error(`Unconfirmed shard owner cleanup failed closed (exit=${completed.exitCode}, staleDetected=${report.staleDetected}, cleanupConfirmed=${report.cleanupConfirmed}, liveOwners=${report.liveOwners?.length ?? "invalid"}, remainingStaleOwners=${report.remainingStaleOwners?.length ?? "invalid"})`); +async function beforeDownload(lease) { + if (process.env.TV_TEST_RUNNER_SELFTEST !== "1" || !process.env.TV_BLAXEL_DOWNLOAD_GATE) return; + const gate = process.env.TV_BLAXEL_DOWNLOAD_GATE; + fs.mkdirSync(gate, { recursive: true }); + fs.writeFileSync(path.join(gate, `shard-${lease.shardIndex}.json`), JSON.stringify({ pid: process.pid, lockId: lease.lockId, sandbox: lease.sandbox.metadata.name, scratch: lease.input.scratch })); + while (!shuttingDown && !fs.existsSync(path.join(gate, "continue"))) { + if (lease.lost) throw lease.lost; + await new Promise((resolve) => setTimeout(resolve, 100)); } - if (lease.logPath) appendLog(lease.logPath, `\n[coordinator] unconfirmed execution owner cleanup: ${JSON.stringify(report)}\n`); - return report; +} + +async function stopAndReleaseLease(lease, { killProcess }) { + if (lease.released) return; + if (lease.finishing) return lease.finishing; + lease.finishing = (async () => { + try { + await lease.pendingWrites; + await lease.dispatchRequest?.catch(() => {}); + // Even a successful named process wait is not a descendant audit. + await confirmTeardown(lease); + if (killProcess && lease.workPromise) await lease.workPromise; + if (!lease.executionRequested) lease.retrieved = true; + if (!lease.retrieved) throw new Error("results not fully retained; keeping lease for recovery"); + await leaseOperation(lease, "release", { retrieved: true }); + lease.released = true; + } finally { await lease.stopRenewal?.(); } + })(); + return lease.finishing; } function buildInterruptedReportSnapshot(signal) { @@ -1160,7 +1141,7 @@ function installSignalHandlers({ killProcesses }) { await Promise.allSettled(leases.map((lease) => stopAndReleaseLease(lease, { killProcess: killProcesses }))); for (const lease of leases) activeLeases.delete(lease.lockId); }, - finalizeReport: () => writeInterruptedReportSnapshot(interruptedSnapshot), + finalizeReport: () => writeInterruptedReportSnapshot(buildInterruptedReportSnapshot(signal) ?? interruptedSnapshot), }); if (reportError) console.error(`Failed to preserve interrupted Blaxel report: ${reportError.stack || reportError.message || reportError}`); if (releaseError) console.error(`Blaxel shard cleanup failed: ${releaseError.stack || releaseError.message || releaseError}`); @@ -1179,13 +1160,14 @@ function listPoolSandboxes() { function resolveTimeouts(profile, overrides) { const profiles = { - normal: { workerTimeoutSeconds: 240, lockTtlSeconds: 900 }, + normal: { workerTimeoutSeconds: 240, lockTtlSeconds: 1200 }, cold: { workerTimeoutSeconds: 480, lockTtlSeconds: 1200 }, repair: { workerTimeoutSeconds: 480, lockTtlSeconds: 1200 }, }; const selected = { ...profiles[profile] }; if (overrides["worker-timeout-seconds"]) selected.workerTimeoutSeconds = Number.parseInt(overrides["worker-timeout-seconds"], 10); if (overrides["lock-ttl-seconds"]) selected.lockTtlSeconds = Number.parseInt(overrides["lock-ttl-seconds"], 10); + if (!Number.isSafeInteger(selected.lockTtlSeconds) || selected.lockTtlSeconds <= 0) fail("--lock-ttl-seconds must be a positive integer"); return selected; } diff --git a/scripts/test/blaxel-lease-lifecycle.d.mts b/scripts/test/blaxel-lease-lifecycle.d.mts new file mode 100644 index 00000000..ab8e5dd3 --- /dev/null +++ b/scripts/test/blaxel-lease-lifecycle.d.mts @@ -0,0 +1,2 @@ +export function renewLease(options: {renew: () => Promise; intervalMs: number; onLost: (error: Error) => void}): () => Promise; +export function finishLease(options: {settleWrites: () => Promise; teardown: () => Promise; download: () => Promise; release: () => Promise; assertOwned: () => void; attempts?: number}): Promise; diff --git a/scripts/test/blaxel-lease-lifecycle.mjs b/scripts/test/blaxel-lease-lifecycle.mjs new file mode 100644 index 00000000..37abdf0d --- /dev/null +++ b/scripts/test/blaxel-lease-lifecycle.mjs @@ -0,0 +1,29 @@ +/** Coordinator-local lifecycle; transport operations are supplied at one seam. */ +export function renewLease({ renew, intervalMs, onLost }) { + let stopped = false; + let pending = Promise.resolve(); + const tick = () => { + if (stopped) return; + pending = renew().catch((error) => { stopped = true; clearInterval(timer); onLost(error); }); + pending.finally(() => { if (!stopped) timer = setTimeout(tick, intervalMs); }); + }; + let timer = setTimeout(tick, intervalMs); + return async () => { stopped = true; clearTimeout(timer); await pending; }; +} + +export async function finishLease({ settleWrites, teardown, download, release, assertOwned, attempts = 3 }) { + await settleWrites(); + assertOwned(); + // Failed teardown still allows evidence retrieval, but can never release. + let teardownError = null; + try { await teardown(); } catch (error) { teardownError = error; } + let retrievalError = null; + for (let attempt = 0; attempt < attempts; attempt++) { + assertOwned(); + try { await download(); retrievalError = null; break; } catch (error) { retrievalError = error; } + } + if (teardownError) throw teardownError; + if (retrievalError) throw retrievalError; + assertOwned(); + await release(); +} diff --git a/scripts/test/blaxel-pool-lifecycle.d.mts b/scripts/test/blaxel-pool-lifecycle.d.mts new file mode 100644 index 00000000..5632ee68 --- /dev/null +++ b/scripts/test/blaxel-pool-lifecycle.d.mts @@ -0,0 +1,3 @@ +export interface Worker {metadata: {name: string; labels: Record}; wait: () => Promise} +export function publishWorker(worker: Worker, operations: {provision: (worker: Worker) => Promise; verify: (worker: Worker) => Promise; update: (name: string, metadata: {labels: Record}) => Promise}): Promise; +export function deleteWorker(worker: Worker, operations: {reserve: (worker: Worker) => Promise; update: (name: string, metadata: {labels: Record}) => Promise; remove: (name: string) => Promise}): Promise; diff --git a/scripts/test/blaxel-pool-lifecycle.mjs b/scripts/test/blaxel-pool-lifecycle.mjs new file mode 100644 index 00000000..10ced551 --- /dev/null +++ b/scripts/test/blaxel-pool-lifecycle.mjs @@ -0,0 +1,14 @@ +/** The manager's SDK seam; fresh/unavailable states retain pool identity. */ +export async function publishWorker(sandbox, { provision, verify, update }) { + await sandbox.wait(); + await provision(sandbox); + await verify(sandbox); + return update(sandbox.metadata.name, { labels: { ...sandbox.metadata.labels, purpose: "testshards" } }); +} + +export async function deleteWorker(sandbox, { reserve, update, remove }) { + // A fresh sandbox was never eligible, so no coordinator can own its checkout. + if (sandbox.metadata.labels.purpose !== "testshards-provisioning") await reserve(sandbox); + await update(sandbox.metadata.name, { labels: { ...sandbox.metadata.labels, purpose: "testshards-unavailable" } }); + await remove(sandbox.metadata.name); +} diff --git a/scripts/test/blaxel-pool.mjs b/scripts/test/blaxel-pool.mjs index ab385c43..cfedce7c 100644 --- a/scripts/test/blaxel-pool.mjs +++ b/scripts/test/blaxel-pool.mjs @@ -5,7 +5,7 @@ export async function listBlaxelPoolSandboxes({ pool, arch, deployedOnly = false return sandboxes.filter((sandbox) => { const labels = sandbox.metadata?.labels ?? {}; return labels.project === "television" - && labels.purpose === "testshards" + && (deployedOnly ? labels.purpose === "testshards" : ["testshards", "testshards-provisioning", "testshards-unavailable"].includes(labels.purpose)) && labels.pool === pool && labels.arch === arch && (!deployedOnly || sandbox.status === "DEPLOYED"); diff --git a/scripts/test/blaxel-worker-lease.d.mts b/scripts/test/blaxel-worker-lease.d.mts new file mode 100644 index 00000000..92762a0c --- /dev/null +++ b/scripts/test/blaxel-worker-lease.d.mts @@ -0,0 +1,11 @@ +import type { Stats } from "node:fs"; +export const DEFAULT_RECOVERY_SECONDS: number; +export const SHARD_OWNER_ENV: string; +export interface LeaseInput { lockId: string; runId?: string; shardIndex?: number; shardTotal?: number; attempt?: number; attemptId?: string; owner?: string; ttlSeconds?: number; expiresAt?: number; tmpRoot?: string; resultsRoot?: string; scratch?: string; auditTokens?: string[]; audit?: boolean; operation?: string; processName?: string; pid?: number; pidFromEnv?: boolean; retrieved?: boolean; worker?: string } +export interface CleanupReport { ok: boolean; before: {availableBytes: number; freeBytes: number}; after: {availableBytes: number; freeBytes: number}; removed: Record; skips: unknown[]; errors: unknown[] } +export interface LeaseRecord { lockId: string; runId: string; attemptId: string; recoverAfter: number; lastRenewal: number; expiresAt: number; recovery: {discardedReports: boolean} | null } +export function shardScratchPath(input: LeaseInput): string; +export function workerLeaseCommand(input: LeaseInput): string; +export function workerEntryCommand(input: LeaseInput, script: string): string; +export function workerLeaseOperation(input: LeaseInput, hooks?: {now?: () => number; scanOwners?: () => Promise>>; stopOwners?: (token: string) => Promise; cleanTemporary?: (input: {tmpRoot: string; current: string | null; onlyCurrent: boolean}) => Promise}): Promise<{ok: boolean; reason?: string; deleting?: boolean; record: LeaseRecord}>; +export function cleanWorkerTemporaryDirectories(input: {tmpRoot: string; current: string | null; onlyCurrent?: boolean}, probes?: {lstat?: (file: string) => Stats; remove?: (file: string) => void}): Promise; diff --git a/scripts/test/blaxel-worker-lease.mjs b/scripts/test/blaxel-worker-lease.mjs index f75a33ba..5217801f 100644 --- a/scripts/test/blaxel-worker-lease.mjs +++ b/scripts/test/blaxel-worker-lease.mjs @@ -1,17 +1,259 @@ -export function workerLeaseCommand({ lockId, runId, shardIndex, shardTotal, owner, expiresAt, tmpRoot = "/tmp" }) { - const freshJson = JSON.stringify({ lockId, runId, shard: `${shardIndex}/${shardTotal}`, owner, expiresAt }); - const reclaimedJson = JSON.stringify({ lockId, runId, shard: `${shardIndex}/${shardTotal}`, owner, expiresAt, reclaimed: true }); - const command = [ - "set -euo pipefail", - "lock=/tmp/tv-testshard.lock", - "now=$(date +%s)", - `if mkdir "$lock" 2>/dev/null; then node -e ${shellQuote(`const fs=require("fs"); const data=${JSON.stringify(freshJson)}; const obj=JSON.parse(data); obj.acquiredAt=Number(process.env.now||${Date.now()}); fs.writeFileSync("/tmp/tv-testshard.lock/owner.json", JSON.stringify(obj));`)}; exit 0; fi`, - "expires=0", - "if [ -f \"$lock/owner.json\" ]; then expires=$(node -e 'try{console.log(JSON.parse(require(\"fs\").readFileSync(process.argv[1],\"utf8\")).expiresAt||0)}catch{console.log(0)}' \"$lock/owner.json\"); fi", - `if [ "$expires" -gt 0 ] && [ "$expires" -lt "$now" ]; then rm -rf "$lock"; mkdir "$lock"; node -e ${shellQuote(`const fs=require("fs"); const data=${JSON.stringify(reclaimedJson)}; const obj=JSON.parse(data); obj.acquiredAt=Number(process.env.now||${Date.now()}); fs.writeFileSync("/tmp/tv-testshard.lock/owner.json", JSON.stringify(obj));`)}; exit 0; fi`, - "exit 42", - ].join("; "); - return command.replaceAll("/tmp/tv-testshard.lock", `${tmpRoot}/tv-testshard.lock`); +import { createHash } from "node:crypto"; + +export const DEFAULT_RECOVERY_SECONDS = 1200; +export const SHARD_OWNER_ENV = "TV_TEST_SHARD_OWNER"; + +export function shardScratchPath({ tmpRoot = "/tmp", lockId, shardIndex, attempt = 1 }) { + const digest = createHash("sha256").update(lockId).digest("hex").slice(0, 20); + return `${tmpRoot}/tv-testshards/r-${digest}-s${shardIndex}-a${attempt}`; +} + +// This code runs before checkout, including after another revision ran here. +// Serialize the actual implementation; never read a helper from the worker clone. +export function workerLeaseCommand(input) { + const tmpRoot = input.tmpRoot ?? "/tmp"; + const source = `${cleanWorkerTemporaryDirectories.toString()}\n${workerLeaseOperation.toString()}\ntry { const result = await workerLeaseOperation(JSON.parse(Buffer.from(process.argv[1], 'base64url'))); console.log(JSON.stringify(result)); if (!result.ok) process.exitCode = 42; } catch (error) { console.error(error.message); process.exitCode = 2; }`; + return `flock -w 20 -o ${quote(`${tmpRoot}/tv-testshard.guard`)} node --input-type=module -e ${quote(source)} ${quote(Buffer.from(JSON.stringify({ ...input, scratch: input.scratch ?? shardScratchPath(input) })).toString("base64url"))}`; } -function shellQuote(value) { return `'${String(value).replaceAll("'", `'"'"'`)}'`; } +export function workerEntryCommand(input, script) { + // exec keeps the registered shell PID/start identity. The token exists at + // process creation, so procfs can observe it even before entry registration. + const register = workerLeaseCommand({ ...input, operation: "enter", pidFromEnv: true }); + return `env TV_TEST_SHARD_OWNER=${quote(input.lockId)} bash -c ${quote(`export TV_SHARD_ENTRY_PID=$$; ${register} && exec bash ${quote(script)}`)}`; +} + +function quote(value) { return `'${String(value).replaceAll("'", `'"'"'`)}'`; } + +/** Must run under the short flock guard. Hooks substitute only declared seams. */ +export async function workerLeaseOperation(input, hooks = {}) { + const fs = await import("node:fs"); + const path = await import("node:path"); + const now = hooks.now ?? Date.now; + const tmpRoot = input.tmpRoot ?? "/tmp"; + const resultsRoot = input.resultsRoot ?? "/workspace/television/.testshards/results"; + const lock = path.join(tmpRoot, "tv-testshard.lock"); + const ownerFile = path.join(lock, "owner.json"); + const operation = input.operation ?? "acquire"; + const ttlMs = (input.ttlSeconds ?? 1200) * 1000; + if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0) throw new Error("invalid recovery interval"); + if (!/^[A-Za-z0-9_.:-]+$/.test(input.lockId ?? "")) throw new Error("invalid lock identity"); + if (fs.lstatSync(tmpRoot).isSymbolicLink()) throw new Error("temporary root is a symlink"); + if (fs.existsSync(lock) && (!fs.lstatSync(lock).isDirectory() || fs.lstatSync(lock).isSymbolicLink())) throw new Error("unsafe lock root"); + let record = null; + const exists = fs.existsSync(lock); + try { record = JSON.parse(fs.readFileSync(ownerFile, "utf8")); } catch (error) { if (!exists && error.code !== "ENOENT") throw error; } + const valid = record && typeof record.lockId === "string" && typeof record.runId === "string" && typeof record.attemptId === "string" && Number.isFinite(record.lastRenewal) && Number.isFinite(record.recoverAfter); + const expired = valid && record.recoverAfter <= now(); + const held = valid ? !expired : Number.isFinite(record?.expiresAt) && record.expiresAt > now() / 1000; + const write = (value) => { + fs.mkdirSync(lock, { recursive: true }); + fs.writeFileSync(`${ownerFile}.next`, JSON.stringify(value)); + fs.renameSync(`${ownerFile}.next`, ownerFile); + }; + const denied = (reason) => ({ ok: false, reason }); + + // Inspect all tokens, including malformed tokens, conservatively. No arguments + // or environments enter the evidence. Kernel threads and zombies own no work. + function readProcess(pid) { + const base = `/proc/${pid}`; + const stat = fs.readFileSync(`${base}/stat`, "utf8"); + const fields = stat.slice(stat.lastIndexOf(")") + 1).trim().split(/\s+/); + if (fields.length < 20) throw new Error("unreadable process identity"); + if (["Z", "X"].includes(fields[0])) return null; + const environment = fs.readFileSync(`${base}/environ`, "utf8").split("\0"); + const value = (key) => environment.find((entry) => entry.startsWith(`${key}=`))?.slice(key.length + 1); + const shardToken = value("TV_TEST_SHARD_OWNER"); + const surfaceToken = value("TV_TEST_SURFACE_OWNER"); + if (!shardToken && !surfaceToken) return null; + if (input.auditTokens && !input.auditTokens.includes(shardToken) && !input.auditTokens.includes(surfaceToken)) return null; + let command = "unavailable"; + try { command = path.basename(fs.readlinkSync(`${base}/exe`)); } catch (error) { if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; } + return { pid, startTime: fields[19], shardToken, surfaceToken, command }; + } + async function scan() { + const owners = []; + for (const name of fs.readdirSync("/proc")) { + if (!/^\d+$/.test(name)) continue; + try { const entry = readProcess(Number(name)); if (entry) owners.push(entry); } + catch (error) { if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; } + } + return owners; + } + const scanOwners = hooks.scanOwners ?? scan; + const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + async function stop(token) { + const actions = new Map(); + const signal = async (name) => { + for (const candidate of (await scanOwners()).filter((entry) => entry.shardToken === token)) { + let current; + try { current = readProcess(candidate.pid); } catch (error) { if (["ENOENT", "ESRCH"].includes(error.code)) continue; throw error; } + if (!current || current.startTime !== candidate.startTime || current.shardToken !== token) continue; + try { process.kill(current.pid, name); } + catch (error) { if (error.code === "ESRCH") continue; throw error; } + actions.set(`${current.pid}:${current.startTime}`, { pid: current.pid, startTime: current.startTime, command: current.command, outcome: name }); + } + }; + await signal("SIGTERM"); + const deadline = Date.now() + 2000; + while ((await scanOwners()).some((entry) => entry.shardToken === token) && Date.now() < deadline) await pause(50); + await signal("SIGKILL"); + const killDeadline = Date.now() + 2000; + while ((await scanOwners()).some((entry) => entry.shardToken === token) && Date.now() < killDeadline) await pause(50); + if ((await scanOwners()).some((entry) => entry.shardToken === token)) throw new Error("owned processes survived cleanup"); + return [...actions.values()]; + } + const stopOwners = hooks.stopOwners ?? stop; + const cleanTemporary = hooks.cleanTemporary ?? cleanWorkerTemporaryDirectories; + const clean = async (current, onlyCurrent = false) => { + const result = await cleanTemporary({ tmpRoot, current, onlyCurrent }); + if (!result.ok) throw new Error(`temporary cleanup incomplete: ${JSON.stringify(result)}`); + return result; + }; + const auditClean = async () => (await scanOwners()).length === 0; + + if (["acquire", "delete"].includes(operation)) { + if (record?.deleting) return operation === "delete" ? { ok: true, deleting: true } : denied("worker reserved for deletion"); + if (exists && (held || input.audit)) return denied("worker lease held"); + let recovery = null; + if (operation === "delete" && valid && expired) { + // Deleting the sandbox is itself cleanup; do not require successful signals. + } else if (valid && expired) { + const processes = await stopOwners(record.lockId); + if (!await auditClean()) return denied("other live or stale owners remain"); + await clean(null); + recovery = { kind: "recovery", previous: { runId: record.runId, lockId: record.lockId, worker: input.worker ?? "unknown", lastRenewal: record.lastRenewal, recoverAfter: record.recoverAfter }, cleanup: "confirmed", discardedReports: true, processes: processes.slice(0, 128) }; + } else if (!input.audit && !await auditClean()) return denied("live or stale owners remain"); + if (operation === "delete") { + write({ lockId: input.lockId, deleting: true }); + return { ok: true, deleting: true }; + } + if (recovery) fs.rmSync(resultsRoot, { recursive: true, force: true }); + const acquiredAt = now(); + const scratch = input.scratch; + if (!scratch || path.dirname(scratch) !== path.join(tmpRoot, "tv-testshards")) throw new Error("invalid scratch path"); + const parent = path.dirname(scratch); + if (fs.existsSync(parent) && fs.lstatSync(parent).isSymbolicLink()) throw new Error("scratch root is a symlink"); + fs.mkdirSync(path.join(scratch, "tmp"), { recursive: true, mode: 0o700 }); + record = { lockId: input.lockId, runId: input.runId, shardIndex: input.shardIndex, attemptId: input.attemptId, owner: input.owner, acquiredAt, lastRenewal: acquiredAt, recoverAfter: acquiredAt + ttlMs, expiresAt: 0, scratch, audit: Boolean(input.audit), dispatch: null, recovery }; + write(record); + return { ok: true, record }; + } + + if (!valid || record.lockId !== input.lockId || expired || record.deleting) return denied("lease lost or expired"); + if (operation === "check") { + return { ok: true, record }; + } else if (operation === "renew") { + record.lastRenewal = now(); record.recoverAfter = record.lastRenewal + ttlMs; write(record); + } else if (operation === "prepare") { + if (record.dispatch || record.prepared) return denied("attempt already prepared or dispatched"); + if (!record.audit && !await auditClean()) return denied("worker audit not clean"); + const cleanup = record.audit ? null : await clean(record.scratch); + fs.rmSync(resultsRoot, { recursive: true, force: true }); + fs.mkdirSync(resultsRoot, { recursive: true }); + fs.writeFileSync(path.join(resultsRoot, "attempt.json"), JSON.stringify({ runId: record.runId, shardIndex: record.shardIndex, attemptId: record.attemptId, lockId: record.lockId })); + fs.writeFileSync(path.join(resultsRoot, "temporary-cleanup.json"), JSON.stringify(cleanup)); + if (record.recovery) { + // First evidence in a freshly reset tree. The bounded receipt is also + // accounted for by the surface writer's shared attempt budget. + const receipt = { schemaVersion: 1, ...record.recovery, identity: { runId: record.runId, shardIndex: record.shardIndex, attemptId: record.attemptId, surfaceId: "worker-recovery" } }; + const raw = JSON.stringify(receipt); + if (Buffer.byteLength(raw) > 128 * 1024) throw new Error("recovery receipt exceeds sidecar budget"); + fs.mkdirSync(path.join(resultsRoot, "resources")); + fs.writeFileSync(path.join(resultsRoot, "resources/recovery.json"), raw); + } + record.prepared = true; write(record); + } else if (operation === "dispatch") { + if (!record.prepared || record.dispatch) return denied("invalid dispatch state"); + record.dispatch = { processName: input.processName, requestedAt: now() }; write(record); + } else if (operation === "enter") { + if (!record.dispatch || record.dispatch.processName !== input.processName || record.dispatch.pid) return denied("dispatch not current"); + const pid = input.pidFromEnv ? Number(process.env.TV_SHARD_ENTRY_PID) : input.pid; + const entry = readProcess(pid); + if (!entry || entry.shardToken !== record.lockId) return denied("entry attribution missing"); + record.dispatch.pid = entry.pid; record.dispatch.startTime = entry.startTime; write(record); + } else if (operation === "teardown") { + // A request that never registered may still arrive. Keep it excluded until + // recovery, whose entry check will reject that old identity. + if (record.dispatch && !record.dispatch.pid) return denied("dispatch outcome unknown"); + const processes = await stopOwners(record.lockId); + if (!record.audit && !await auditClean()) return denied("worker audit not clean"); + if (record.prepared) { + const setupFile = path.join(resultsRoot, `setup-shard-${record.shardIndex}.json`); + if (!fs.existsSync(setupFile)) { + const inputFile = path.join(record.scratch, "setup.jsonl"); + const rows = fs.existsSync(inputFile) ? fs.readFileSync(inputFile, "utf8").split("\n").filter(Boolean).map((line) => JSON.parse(line)) : []; + fs.writeFileSync(setupFile, JSON.stringify(rows)); + } + const summaryFile = path.join(resultsRoot, `shard-${record.shardIndex}.json`); + if (!fs.existsSync(summaryFile)) fs.writeFileSync(summaryFile, JSON.stringify({ status: "infra-failed", failureKind: "worker-execution", failureStep: "interrupted-before-summary", failureMessage: "The owned process ended without a finalized shard summary.", shardIndex: record.shardIndex, tasks: [], neverProduced: ["final shard summary"] })); + } + const cleanup = await clean(record.scratch, true); + record.teardown = { confirmed: true, processes, cleanup }; write(record); + if (record.prepared) { + fs.writeFileSync(path.join(resultsRoot, "teardown.json"), JSON.stringify(record.teardown)); + const files = []; + const inventory = (dir) => { for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const file = path.join(dir, entry.name); + if (entry.isSymbolicLink()) throw new Error("symlink in retained evidence"); + if (entry.isDirectory()) inventory(file); + else if (entry.name !== "evidence.json") files.push(path.relative(resultsRoot, file)); + } }; + inventory(resultsRoot); + fs.writeFileSync(path.join(resultsRoot, "evidence.json"), JSON.stringify({ runId: record.runId, attemptId: record.attemptId, files })); + } + } else if (operation === "release") { + if (!record.teardown?.confirmed || !input.retrieved) return denied("teardown and retained evidence required"); + fs.rmSync(lock, { recursive: true }); + } else throw new Error(`unknown lease operation ${operation}`); + return { ok: true, record }; +} + +/** Direct allowed directories only; lstat traversal never follows symlinks. */ +export async function cleanWorkerTemporaryDirectories({ tmpRoot = "/tmp", current, onlyCurrent = false }, probes = {}) { + const fs = await import("node:fs"); + const path = await import("node:path"); + const lstat = probes.lstat ?? fs.lstatSync; + const remove = probes.remove ?? ((file) => fs.rmSync(file, { recursive: true, force: false })); + const scratchRoot = path.join(tmpRoot, "tv-testshards"); + const free = () => { const value = fs.statfsSync(tmpRoot); return { availableBytes: value.bavail * value.bsize, freeBytes: value.bfree * value.bsize }; }; + const report = { ok: true, before: free(), after: null, removed: {}, skips: [], errors: [] }; + for (const root of [tmpRoot, scratchRoot]) { + if (!fs.existsSync(root)) continue; + if (lstat(root).isSymbolicLink() || !lstat(root).isDirectory()) throw new Error(`unsafe cleanup root ${root}`); + if (root !== tmpRoot && lstat(root).dev !== lstat(tmpRoot).dev) throw new Error("scratch root crosses a mount boundary"); + } + function allocated(file, device) { + const stat = lstat(file); + if (stat.dev !== device) throw new Error("unexpected mount boundary"); + let bytes = stat.blocks * 512; + if (stat.isDirectory()) for (const name of fs.readdirSync(file)) bytes += allocated(path.join(file, name), device); + return bytes; + } + const candidates = []; + if (onlyCurrent) { + if (!current || path.dirname(current) !== scratchRoot) throw new Error("current scratch escapes allowed root"); + candidates.push([current, "current"]); + } else { + for (const name of fs.readdirSync(tmpRoot)) { + const prefix = ["television-source-cli-e2e-", "television-web-bundle-"].find((value) => name.startsWith(value)); + if (prefix) candidates.push([path.join(tmpRoot, name), prefix]); + } + if (fs.existsSync(scratchRoot)) for (const name of fs.readdirSync(scratchRoot)) { + const file = path.join(scratchRoot, name); if (file !== current) candidates.push([file, "stale-scratch"]); + } + } + for (const [file, prefix] of candidates) { + try { + if (!fs.existsSync(file)) continue; + const stat = lstat(file); + if (!stat.isDirectory() || stat.isSymbolicLink()) { report.skips.push({ prefix, reason: "not a real directory" }); continue; } + const bytes = allocated(file, lstat(tmpRoot).dev); + remove(file); + const counts = report.removed[prefix] ??= { directories: 0, allocatedBytes: 0 }; + counts.directories++; counts.allocatedBytes += bytes; + } catch (error) { report.ok = false; report.errors.push({ prefix, reason: error.code ?? error.message }); } + } + report.after = free(); + return report; +} diff --git a/scripts/test/cli.mjs b/scripts/test/cli.mjs index ac1aba81..7800d314 100644 --- a/scripts/test/cli.mjs +++ b/scripts/test/cli.mjs @@ -480,6 +480,7 @@ async function runBlaxel(selectionOptions, surfaces, testRetries = 0) { const args = ["scripts/run-blaxel-testshards.mjs", "--pool", selectionOptions.pool ?? "poc", "--commit", selectionOptions.commit ?? "HEAD", "--output-dir", outputDir]; if (selectionOptions["timeout-profile"]) args.push("--timeout-profile", selectionOptions["timeout-profile"]); else args.push("--timeout-profile", "cold"); + if (selectionOptions["lock-ttl-seconds"] !== undefined) args.push("--lock-ttl-seconds", selectionOptions["lock-ttl-seconds"]); if (selectionOptions.shards) args.push("--shards", selectionOptions.shards); if (selectionOptions["shard-indices"]) args.push("--shard-indices", selectionOptions["shard-indices"]); if (selectionOptions["allow-shard-count-override"]) args.push("--allow-shard-count-override", selectionOptions["allow-shard-count-override"]); diff --git a/scripts/test/resource-diagnostics.mjs b/scripts/test/resource-diagnostics.mjs index c83a64fa..ee138bba 100644 --- a/scripts/test/resource-diagnostics.mjs +++ b/scripts/test/resource-diagnostics.mjs @@ -1,3 +1,4 @@ +import { BASELINE_TIMING_PROVIDER } from "./timing-baseline.mjs"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -19,7 +20,7 @@ const warnDefault = (message) => console.error(`[resource diagnostics] ${message /** Shared by planned tasks and the targeted surface parent, never the native child. */ export function surfaceResourceDiagnostics({ resultsDir, surfaceId, env = process.env, runId }) { - if (env.TV_TEST_TIMING_PROVIDER !== "blaxel-playwright-x64-4vcpu") return { file: null, finish: () => [] }; + if (env.TV_TEST_TIMING_PROVIDER !== BASELINE_TIMING_PROVIDER) return { file: null, finish: () => [] }; return startResourceDiagnostics({ root: resultsDir, identity: { runId: env.TV_TEST_RUN_ID ?? runId, shardIndex: Number(env.TV_TEST_SHARD_INDEX ?? "1"), diff --git a/scripts/test/surface-supervisor.mjs b/scripts/test/surface-supervisor.mjs index 508be8aa..368962bd 100644 --- a/scripts/test/surface-supervisor.mjs +++ b/scripts/test/surface-supervisor.mjs @@ -59,7 +59,7 @@ export class SurfaceSupervisor { } childEnv(env = process.env) { - return { ...env, [OWNER_TOKEN_ENV]: this.ownerToken }; + return { ...env, TMPDIR: env.TMPDIR ?? process.env.TMPDIR, ...(process.env.TV_TEST_SHARD_OWNER ? { TV_TEST_SHARD_OWNER: process.env.TV_TEST_SHARD_OWNER } : {}), [OWNER_TOKEN_ENV]: this.ownerToken }; } run(command, args = [], { diff --git a/test/helpers/owned-process.ts b/test/helpers/owned-process.ts index 4410672f..ac0ed595 100644 --- a/test/helpers/owned-process.ts +++ b/test/helpers/owned-process.ts @@ -35,8 +35,10 @@ export function spawnOwnedProcess(command: string, args: readonly string[] = [], const { termGraceMs = TEST_PROCESS_TERM_GRACE_MS, env: requestedEnv, ...spawnOptions } = options; if (!Number.isFinite(termGraceMs) || termGraceMs < 0) throw new Error("termGraceMs must be a non-negative finite number"); const env: NodeJS.ProcessEnv = { ...(requestedEnv ?? process.env) }; - const ownerToken = process.env[OWNER_TOKEN_ENV]; - if (ownerToken !== undefined) env[OWNER_TOKEN_ENV] = ownerToken; + for (const key of [OWNER_TOKEN_ENV, "TV_TEST_SHARD_OWNER"]) { + if (process.env[key] !== undefined) env[key] = process.env[key]; + } + env.TMPDIR ??= process.env.TMPDIR; const detached = process.platform !== "win32"; const child = spawn(command, [...args], { ...spawnOptions, env, detached }); if (!child.pid) { diff --git a/test/repo/blaxel-lease-contracts.test.ts b/test/repo/blaxel-lease-contracts.test.ts new file mode 100644 index 00000000..cdfbb1bb --- /dev/null +++ b/test/repo/blaxel-lease-contracts.test.ts @@ -0,0 +1,116 @@ +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, existsSync, symlinkSync, lstatSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { workerLeaseOperation, cleanWorkerTemporaryDirectories } from "../../scripts/test/blaxel-worker-lease.mjs"; +import { finishLease } from "../../scripts/test/blaxel-lease-lifecycle.mjs"; +import { publishWorker, deleteWorker } from "../../scripts/test/blaxel-pool-lifecycle.mjs"; + +// Contract clocks, owner audits/signals and SDK operations are substitutions. +// Filesystem assertions cross the real cleaner; the live driver proves transport. +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +function fixture(record?: unknown) { + const tmpRoot = mkdtempSync(path.join(os.tmpdir(), "lease-contract-")); roots.push(tmpRoot); + const ownerFile = path.join(tmpRoot, "tv-testshard.lock/owner.json"); + if (record !== undefined) { mkdirSync(path.dirname(ownerFile)); writeFileSync(ownerFile, typeof record === "string" ? record : JSON.stringify(record)); } + const input = { lockId: "incoming", runId: "incoming-run", shardIndex: 1, attemptId: "incoming-attempt", tmpRoot, scratch: path.join(tmpRoot, "tv-testshards/current"), resultsRoot: path.join(tmpRoot, "results") }; + return { tmpRoot, input, ownerFile }; +} +const expired = { lockId: "old", runId: "old-run", attemptId: "old-attempt", lastRenewal: 1, recoverAfter: 2, expiresAt: 0 }; + +test("expired active and quiet leases use the same cleanup; uncertain owners hold", async () => { + for (const active of [false, true]) { + const f = fixture(expired); const calls: string[] = []; + let owners = active ? [{ shardToken: "old" }] : []; + const result = await workerLeaseOperation(f.input, { now: () => 3000, scanOwners: async () => owners, stopOwners: async (token: string) => { calls.push(token); owners = []; return []; } }); + expect(result.ok).toBe(true); expect(calls).toEqual(["old"]); + expect(result.record.recovery.discardedReports).toBe(true); + } + const f = fixture(expired); + const result = await workerLeaseOperation(f.input, { now: () => 3000, stopOwners: async () => [], scanOwners: async () => [{ shardToken: "unrelated" }] }); + expect(result).toMatchObject({ ok: false }); + expect(JSON.parse(readFileSync(f.ownerFile, "utf8")).lockId).toBe("old"); +}); + +test("legacy, incomplete, unreadable and audit leases fail closed without clean ownership", async () => { + for (const record of [{ lockId: "legacy", expiresAt: 1 }, {}, "unreadable"]) { + for (const operation of ["acquire", "delete"]) { + const f = fixture(record); + expect(await workerLeaseOperation({ ...f.input, operation }, { now: () => 3000, scanOwners: async () => [{}] })).toMatchObject({ ok: false }); + expect((await workerLeaseOperation({ ...f.input, operation }, { now: () => 3000, scanOwners: async () => [] })).ok).toBe(true); + } + } + for (const record of [expired, { lockId: "legacy", expiresAt: 100 }]) { + const f = fixture(record); + expect(await workerLeaseOperation({ ...f.input, audit: true }, { now: () => 3000, scanOwners: async () => { throw Error("must not inspect held worker"); } })).toMatchObject({ ok: false }); + } + const f = fixture({ lockId: "legacy", expiresAt: 100 }); + expect(await workerLeaseOperation(f.input, { now: () => 3000, scanOwners: async () => [] })).toMatchObject({ ok: false }); +}); + +test("cleanup failures keep the old identity, while explicit deletion reserves an expired worker", async () => { + const f = fixture(expired); + await expect(workerLeaseOperation(f.input, { now: () => 3000, scanOwners: async () => [], stopOwners: async () => { throw Error("cannot signal"); } })).rejects.toThrow("cannot signal"); + expect(JSON.parse(readFileSync(f.ownerFile, "utf8")).lockId).toBe("old"); + expect(await workerLeaseOperation({ ...f.input, operation: "delete" }, { now: () => 3000, scanOwners: async () => { throw Error("unavailable"); } })).toMatchObject({ ok: true, deleting: true }); + expect(await workerLeaseOperation(f.input)).toMatchObject({ ok: false }); +}); + +test("only known real directories are removed, with allocated-byte evidence and outside links preserved", async () => { + const f = fixture(); const outside = path.join(f.tmpRoot, "unrelated"); mkdirSync(outside); writeFileSync(path.join(outside, "keep"), "keep"); + for (const name of ["television-source-cli-e2e-old", "television-web-bundle-old", "tv-testshards/stale", "tv-testshards/no-metadata", "tv-testshards/current"]) { mkdirSync(path.join(f.tmpRoot, name), { recursive: true }); writeFileSync(path.join(f.tmpRoot, name, "data"), "payload"); } + symlinkSync(outside, path.join(f.tmpRoot, "television-web-bundle-link")); + symlinkSync(outside, path.join(f.tmpRoot, "television-source-cli-e2e-old/link")); + const result = await cleanWorkerTemporaryDirectories({ tmpRoot: f.tmpRoot, current: f.input.scratch }); + expect(result.ok).toBe(true); + expect(result.removed["stale-scratch"].directories).toBe(2); + expect(result.removed["television-source-cli-e2e-"].allocatedBytes).toBeGreaterThanOrEqual(0); + expect(result.before.availableBytes).toBeGreaterThan(0); expect(result.after.availableBytes).toBeGreaterThan(0); + expect(existsSync(path.join(outside, "keep"))).toBe(true); expect(existsSync(f.input.scratch)).toBe(true); + expect(lstatSync(path.join(f.tmpRoot, "television-web-bundle-link")).isSymbolicLink()).toBe(true); + await cleanWorkerTemporaryDirectories({ tmpRoot: f.tmpRoot, current: f.input.scratch, onlyCurrent: true }); + expect(existsSync(f.input.scratch)).toBe(false); + const linked = path.join(f.tmpRoot, "link"); symlinkSync(outside, linked); + await expect(cleanWorkerTemporaryDirectories({ tmpRoot: linked, current: null })).rejects.toThrow("unsafe cleanup root"); +}); + +test("mount boundaries and incomplete removal remain reported failures without reclaimed counts", async () => { + for (const failure of ["mount", "remove"]) { + const f = fixture(); const dir = path.join(f.tmpRoot, "television-web-bundle-old"); mkdirSync(dir); + const probes = failure === "mount" ? { lstat: (file: string) => { const stat = lstatSync(file); if (file === dir) Object.defineProperty(stat, "dev", { value: stat.dev + 1 }); return stat; } } : { remove: () => { throw Object.assign(Error("incomplete"), { code: "EIO" }); } }; + const result = await cleanWorkerTemporaryDirectories({ tmpRoot: f.tmpRoot, current: null }, probes); + expect(result.ok).toBe(false); expect(result.removed).toEqual({}); expect(result.errors).toHaveLength(1); expect(existsSync(dir)).toBe(true); + } +}); + +test("retrieval retries after teardown without rerunning work; uncertainty never releases", async () => { + const calls: string[] = []; let attempts = 0; + const operations = { settleWrites: async () => { calls.push("writes"); }, assertOwned: () => {}, teardown: async () => { calls.push("teardown"); }, download: async () => { calls.push("download"); if (++attempts === 1) throw Error("transport"); }, release: async () => { calls.push("release"); } }; + await finishLease(operations); + expect(calls).toEqual(["writes", "teardown", "download", "download", "release"]); + for (const broken of ["teardown", "download", "ownership"] as const) { + calls.length = 0; + await expect(finishLease({ ...operations, ...(broken === "teardown" ? { teardown: async () => { throw Error("unknown dispatch"); } } : broken === "download" ? { download: async () => { throw Error("transport"); } } : { assertOwned: () => { throw Error("lost lease"); } }) })).rejects.toThrow(); + expect(calls).not.toContain("release"); + } +}); + +test("pool eligibility follows verification, and deletion reserves then unpublishes", async () => { + const calls: string[] = []; + const sandbox = { metadata: { name: "fixture", labels: { purpose: "testshards-provisioning", pool: "private" } }, wait: async () => { calls.push("wait"); } }; + const update = async (_name: string, metadata: { labels: { purpose: string } }) => { calls.push(metadata.labels.purpose); }; + await publishWorker(sandbox, { provision: async () => { calls.push("provision"); }, verify: async () => { calls.push("verify"); }, update }); + expect(calls).toEqual(["wait", "provision", "verify", "testshards"]); + calls.length = 0; + await expect(publishWorker(sandbox, { provision: async () => {}, verify: async () => { throw Error("wrong architecture"); }, update })).rejects.toThrow(); + expect(calls).not.toContain("testshards"); + sandbox.metadata.labels.purpose = "testshards"; + calls.length = 0; + const ops = { reserve: async () => { calls.push("reserve"); }, update, remove: async () => { calls.push("delete"); throw Error("provider unavailable"); } }; + await expect(deleteWorker(sandbox, ops)).rejects.toThrow(); + expect(calls).toEqual(["reserve", "testshards-unavailable", "delete"]); + calls.length = 0; + await expect(deleteWorker(sandbox, { ...ops, reserve: async () => { throw Error("valid lease"); } })).rejects.toThrow(); + expect(calls).toEqual([]); +}); diff --git a/test/repo/blaxel-worker-lease.test.ts b/test/repo/blaxel-worker-lease.test.ts index 3e6803d7..007c32a8 100644 --- a/test/repo/blaxel-worker-lease.test.ts +++ b/test/repo/blaxel-worker-lease.test.ts @@ -36,3 +36,17 @@ test.skipIf(process.platform !== "linux")("a quiet expired renewable lease recov expect(f.record().lockId).toBe(f.input.lockId); expect(f.record().recovery).toMatchObject({ previous: { lockId: "abandoned", runId: "old-run" }, cleanup: "confirmed", discardedReports: true }); }); + +test.skipIf(process.platform !== "linux")("renewal and release reject old identities, expiry, and missing retrieval", () => { + const f = fixture(); + expect(f.run({ ttlSeconds: 60 }).status).toBe(0); + expect(f.run({ operation: "renew", lockId: "other" }).status).toBe(42); + expect(f.run({ operation: "release", retrieved: true }).status).toBe(42); + expect(f.run({ operation: "teardown" }).status).toBe(0); + expect(f.run({ operation: "release" }).status).toBe(42); + expect(f.run({ operation: "renew", ttlSeconds: 90 }).status).toBe(0); + expect(f.record().recoverAfter - f.record().lastRenewal).toBe(90_000); + writeFileSync(f.recordPath, JSON.stringify({ ...f.record(), recoverAfter: 1 })); + expect(f.run({ operation: "renew" }).status).toBe(42); + expect(f.run({ operation: "release", retrieved: true }).status).toBe(42); +}); From e102e200572a6a8050cf427ebd6f3e6054882530 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:58:53 +0000 Subject: [PATCH 36/64] Exercise worker recovery and private Blaxel qualification --- ...arness-resource-cleanup-review-response.md | 15 ++ scripts/run-blaxel-testshards.mjs | 24 ++- scripts/test/blaxel-lease-acquisition.d.mts | 1 + scripts/test/blaxel-lease-acquisition.mjs | 4 + scripts/test/blaxel-pool.d.mts | 2 + scripts/test/blaxel-worker-lease.mjs | 19 +- scripts/test/cli.mjs | 6 +- test.config.mjs | 4 + test/repo/blaxel-pool-discovery.test.ts | 6 +- test/repo/blaxel-worker-lease.test.ts | 83 ++++++++- test/repo/test-runner-config.test.ts | 12 ++ test/repo/test-runner-provider.test.ts | 43 ++--- .../blaxel-harness/driver/harness.test.ts | 173 ++++++++++++++++++ .../blaxel-harness/driver/vitest.config.ts | 2 + .../blaxel-harness/fixtures/outcomes.test.ts | 7 + .../blaxel-harness/fixtures/support.ts | 20 ++ .../blaxel-harness/fixtures/temporary.test.ts | 33 ++++ .../blaxel-harness/fixtures/vitest.config.ts | 2 + 18 files changed, 411 insertions(+), 45 deletions(-) create mode 100644 scripts/test/blaxel-pool.d.mts create mode 100644 test/runner-fixtures/blaxel-harness/driver/harness.test.ts create mode 100644 test/runner-fixtures/blaxel-harness/driver/vitest.config.ts create mode 100644 test/runner-fixtures/blaxel-harness/fixtures/outcomes.test.ts create mode 100644 test/runner-fixtures/blaxel-harness/fixtures/support.ts create mode 100644 test/runner-fixtures/blaxel-harness/fixtures/temporary.test.ts create mode 100644 test/runner-fixtures/blaxel-harness/fixtures/vitest.config.ts diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 01e81fbb..f63a4c2c 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -260,3 +260,18 @@ process cleanup, never reuse beneath live work. The converged specs govern that clarification. The supervisor explicitly directed no further Primitive debugging. Its partial/truncated decision scan is not a complete authority check. No tests, builds or application processes ran on the shared host. + +The next private run (`2c0f5d6b`, +`.test-runs/2026-10-27T18-50-52-346Z-p19842-r74788ac655aa50a6`) did not qualify: +Vitest transformed dynamic imports inside the serialized command, and post-run +cleanup mistook overlay backing-device IDs for mount boundaries. Both workers +retained their leases; no shared workers were used. Direct read-only inspection +showed device 19 for `/tmp` directories and device 20 for regular files, with one +overlay mount in `/proc/self/mountinfo`. Correct the serialization and inspect +actual mount records while retaining directory-boundary rejection. Do not alter +the held deadlines; subsequent work will use ordinary expiry recovery. + +A further Primitive warning named `dec_8e3d38a8`, check +`td7crdek1b1n4dhajyww5p174s8h23bq`. Its decision text was not read. Per the +supervisor's direction, record the unread warning and continue with the converged +ordinary Linux fixture seam and private live-driver proof; do not debug the CLI. diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index 8eaadd97..f638340b 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -328,7 +328,10 @@ async function leaseOperation(lease, operation, extra = {}) { const result = await lease.sandbox.process.exec({ name: `tv-${operation}-${randomUUID()}`, command: `bash -lc ${shellQuote(body)}`, waitForCompletion: true, timeout: 30 }); const lines = chunkToText(result.logs ?? "").trim().split("\n"); let response; - try { response = JSON.parse(lines.at(-1)); } catch { throw new Error(`worker ${operation} returned no ownership evidence (exit ${result.exitCode})`); } + try { response = JSON.parse(lines.at(-1)); } catch { + const details = chunkToText(await lease.sandbox.process.logs(result.name, "all")).slice(-4000); + throw new Error(`worker ${operation} returned no ownership evidence (exit ${result.exitCode}): ${details}`); + } if (result.exitCode !== 0 || !response.ok) throw new Error(`worker ${operation}: ${response.reason ?? "failed"}`); return response; } @@ -398,7 +401,7 @@ function acquisitionOrder(sandboxes, { slot, requestedWorkers }) { } -async function runShard(lease, outputDir, { attempt, onExecutionRequested = () => {}, onExecutionStarted = () => {}, onResultProgress = () => {} } = {}) { +async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onExecutionStarted = () => {}, onResultProgress = () => {} } = {}) { const { sandbox, shardIndex, lockId } = lease; const diagnosticAttemptId = lease.input.attemptId; const startedAt = Date.now(); @@ -573,7 +576,7 @@ async function runShard(lease, outputDir, { attempt, onExecutionRequested = () = shardSummaryError = identityError ?? resultError; appendLog(logPath, `\n[coordinator] ${shardSummaryError}\n`); } - if (!shardSummary || identityError || resultError) throw new Error(shardSummaryError ?? "missing shard summary"); + if (!shardSummary || identityError) throw new Error(shardSummaryError ?? "missing shard summary"); const hasSummaryFile = fs.existsSync(summaryPath); const status = classifyDownloadedShardStatus({ identityError, shardSummary, exitCode, hasSummaryFile, setupTimings }); const completedAt = new Date().toISOString(); @@ -1029,7 +1032,20 @@ async function retainAttemptEvidence(lease, directory) { const root = "/workspace/television/.testshards/results"; const identity = JSON.parse(await lease.sandbox.fs.read(`${root}/attempt.json`)); for (const key of ["lockId", "runId", "shardIndex", "attemptId"]) if (identity[key] !== lease.input[key]) throw new Error(`attempt evidence identity mismatch: ${key}`); - const evidence = JSON.parse(await lease.sandbox.fs.read(`${root}/evidence.json`)); + let evidence; + try { evidence = JSON.parse(await lease.sandbox.fs.read(`${root}/evidence.json`)); } + catch (error) { + if (lease.teardownConfirmed || error?.status !== 404) throw error; + // Teardown uncertainty must not prevent retaining already-produced evidence. + const files = []; + const collect = async (directory) => { + const listing = await lease.sandbox.fs.ls(directory); + for (const file of listing.files ?? []) files.push(path.posix.relative(root, `${directory}/${file.name}`)); + for (const child of listing.subdirectories ?? []) await collect(`${directory}/${child.name}`); + }; + await collect(root); + evidence = { runId: lease.input.runId, attemptId: lease.input.attemptId, files }; + } if (evidence.runId !== lease.input.runId || evidence.attemptId !== lease.input.attemptId || !Array.isArray(evidence.files)) throw new Error("invalid retained-evidence inventory"); for (const name of evidence.files) { if (typeof name !== "string" || name.startsWith("/") || name.split("/").includes("..")) throw new Error("invalid evidence path"); diff --git a/scripts/test/blaxel-lease-acquisition.d.mts b/scripts/test/blaxel-lease-acquisition.d.mts index 70b2c46a..103f5b23 100644 --- a/scripts/test/blaxel-lease-acquisition.d.mts +++ b/scripts/test/blaxel-lease-acquisition.d.mts @@ -22,3 +22,4 @@ export function acquireShardLeases(options: { maxConcurrency: number; acquire: (assignment: { shardIndex: number; slot: number }) => Promise; }): Promise>>; +export function leaseRecoveryArgs(options: Record): unknown[]; diff --git a/scripts/test/blaxel-lease-acquisition.mjs b/scripts/test/blaxel-lease-acquisition.mjs index 4f2d767e..6501d361 100644 --- a/scripts/test/blaxel-lease-acquisition.mjs +++ b/scripts/test/blaxel-lease-acquisition.mjs @@ -83,3 +83,7 @@ export async function acquireShardLeases({ shardIndices, maxConcurrency, acquire function errorMessage(error) { return error instanceof Error ? error.message : String(error); } + +export function leaseRecoveryArgs(options) { + return options["lock-ttl-seconds"] === undefined ? [] : ["--lock-ttl-seconds", options["lock-ttl-seconds"]]; +} diff --git a/scripts/test/blaxel-pool.d.mts b/scripts/test/blaxel-pool.d.mts new file mode 100644 index 00000000..275c21ab --- /dev/null +++ b/scripts/test/blaxel-pool.d.mts @@ -0,0 +1,2 @@ +import type { SandboxInstance } from "@blaxel/core"; +export function listBlaxelPoolSandboxes(options: {pool: string; arch: string; deployedOnly?: boolean}): Promise; diff --git a/scripts/test/blaxel-worker-lease.mjs b/scripts/test/blaxel-worker-lease.mjs index 5217801f..b9327474 100644 --- a/scripts/test/blaxel-worker-lease.mjs +++ b/scripts/test/blaxel-worker-lease.mjs @@ -1,4 +1,6 @@ import { createHash } from "node:crypto"; +import { createRequire } from "node:module"; +const require = createRequire(import.meta.url); export const DEFAULT_RECOVERY_SECONDS = 1200; export const SHARD_OWNER_ENV = "TV_TEST_SHARD_OWNER"; @@ -12,8 +14,8 @@ export function shardScratchPath({ tmpRoot = "/tmp", lockId, shardIndex, attempt // Serialize the actual implementation; never read a helper from the worker clone. export function workerLeaseCommand(input) { const tmpRoot = input.tmpRoot ?? "/tmp"; - const source = `${cleanWorkerTemporaryDirectories.toString()}\n${workerLeaseOperation.toString()}\ntry { const result = await workerLeaseOperation(JSON.parse(Buffer.from(process.argv[1], 'base64url'))); console.log(JSON.stringify(result)); if (!result.ok) process.exitCode = 42; } catch (error) { console.error(error.message); process.exitCode = 2; }`; - return `flock -w 20 -o ${quote(`${tmpRoot}/tv-testshard.guard`)} node --input-type=module -e ${quote(source)} ${quote(Buffer.from(JSON.stringify({ ...input, scratch: input.scratch ?? shardScratchPath(input) })).toString("base64url"))}`; + const source = `${cleanWorkerTemporaryDirectories.toString()}\n${workerLeaseOperation.toString()}\nworkerLeaseOperation(JSON.parse(Buffer.from(process.argv[1], 'base64url'))).then(result => { console.log(JSON.stringify(result)); if (!result.ok) process.exitCode = 42; }).catch(error => { console.error(error.message); process.exitCode = 2; });`; + return `flock -w 20 -o ${quote(`${tmpRoot}/tv-testshard.guard`)} node -e ${quote(source)} ${quote(Buffer.from(JSON.stringify({ ...input, scratch: input.scratch ?? shardScratchPath(input) })).toString("base64url"))}`; } export function workerEntryCommand(input, script) { @@ -27,8 +29,8 @@ function quote(value) { return `'${String(value).replaceAll("'", `'"'"'`)}'`; } /** Must run under the short flock guard. Hooks substitute only declared seams. */ export async function workerLeaseOperation(input, hooks = {}) { - const fs = await import("node:fs"); - const path = await import("node:path"); + const fs = require("node:fs"); + const path = require("node:path"); const now = hooks.now ?? Date.now; const tmpRoot = input.tmpRoot ?? "/tmp"; const resultsRoot = input.resultsRoot ?? "/workspace/television/.testshards/results"; @@ -211,11 +213,12 @@ export async function workerLeaseOperation(input, hooks = {}) { /** Direct allowed directories only; lstat traversal never follows symlinks. */ export async function cleanWorkerTemporaryDirectories({ tmpRoot = "/tmp", current, onlyCurrent = false }, probes = {}) { - const fs = await import("node:fs"); - const path = await import("node:path"); + const fs = require("node:fs"); + const path = require("node:path"); const lstat = probes.lstat ?? fs.lstatSync; const remove = probes.remove ?? ((file) => fs.rmSync(file, { recursive: true, force: false })); const scratchRoot = path.join(tmpRoot, "tv-testshards"); + const mountPoints = new Set(fs.readFileSync("/proc/self/mountinfo", "utf8").trim().split("\n").map((line) => line.split(" ")[4].replace(/\\([0-7]{3})/g, (_match, code) => String.fromCharCode(parseInt(code, 8))))); const free = () => { const value = fs.statfsSync(tmpRoot); return { availableBytes: value.bavail * value.bsize, freeBytes: value.bfree * value.bsize }; }; const report = { ok: true, before: free(), after: null, removed: {}, skips: [], errors: [] }; for (const root of [tmpRoot, scratchRoot]) { @@ -225,7 +228,9 @@ export async function cleanWorkerTemporaryDirectories({ tmpRoot = "/tmp", curren } function allocated(file, device) { const stat = lstat(file); - if (stat.dev !== device) throw new Error("unexpected mount boundary"); + // Overlay files may expose their backing device while directories expose + // overlay's device. A regular file's st_dev alone is not a mount boundary. + if (mountPoints.has(file) || (stat.isDirectory() && stat.dev !== device)) throw new Error("unexpected mount boundary"); let bytes = stat.blocks * 512; if (stat.isDirectory()) for (const name of fs.readdirSync(file)) bytes += allocated(path.join(file, name), device); return bytes; diff --git a/scripts/test/cli.mjs b/scripts/test/cli.mjs index 7800d314..04761b53 100644 --- a/scripts/test/cli.mjs +++ b/scripts/test/cli.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { leaseRecoveryArgs } from "./blaxel-lease-acquisition.mjs"; import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; @@ -480,7 +481,7 @@ async function runBlaxel(selectionOptions, surfaces, testRetries = 0) { const args = ["scripts/run-blaxel-testshards.mjs", "--pool", selectionOptions.pool ?? "poc", "--commit", selectionOptions.commit ?? "HEAD", "--output-dir", outputDir]; if (selectionOptions["timeout-profile"]) args.push("--timeout-profile", selectionOptions["timeout-profile"]); else args.push("--timeout-profile", "cold"); - if (selectionOptions["lock-ttl-seconds"] !== undefined) args.push("--lock-ttl-seconds", selectionOptions["lock-ttl-seconds"]); + args.push(...leaseRecoveryArgs(selectionOptions)); if (selectionOptions.shards) args.push("--shards", selectionOptions.shards); if (selectionOptions["shard-indices"]) args.push("--shard-indices", selectionOptions["shard-indices"]); if (selectionOptions["allow-shard-count-override"]) args.push("--allow-shard-count-override", selectionOptions["allow-shard-count-override"]); @@ -867,6 +868,9 @@ function assertRegistryValid() { } function enforceExecutionPlacement(provider, surfaces) { + if (surfaces.some((surface) => surface.id === "experiment:blaxel-harness")) { + if (provider !== "local" || process.env.TV_TEST_SHARD_OWNER || process.env.TV_BLAXEL_HARNESS_COORDINATOR !== "tv-cleanup-coordinator-rose") fail("The Blaxel harness driver requires its dedicated coordinator outside the worker pool."); + } const isolated = surfaces.filter((surface) => surface.tags.includes("isolated-github-only")); if (isolated.length === 0) return; const ids = isolated.map((surface) => surface.id).join(", "); diff --git a/test.config.mjs b/test.config.mjs index 1b279a3d..4cafb960 100644 --- a/test.config.mjs +++ b/test.config.mjs @@ -92,6 +92,10 @@ export default { executionGroup("agent:root", "agent: root", 14, [ surface("agent:root", "vitest", "test/agent/vitest.config.ts", { kind: "agent", roots: ["test/agent"], agent: true, tags: ["agent"] }), ]), + executionGroup("experiment:blaxel-harness", "experiment: Blaxel harness", 88, [ + surface("experiment:blaxel-harness", "vitest", "test/runner-fixtures/blaxel-harness/driver/vitest.config.ts", { kind: "experiment", roots: ["test/runner-fixtures/blaxel-harness/driver"], tags: ["blaxel-harness-driver"] }), + surface("experiment:blaxel-harness-fixtures", "vitest", "test/runner-fixtures/blaxel-harness/fixtures/vitest.config.ts", { kind: "experiment", roots: ["test/runner-fixtures/blaxel-harness/fixtures"], tags: ["blaxel-harness-fixture"] }), + ]), executionGroup("experiment:dynamic-services", "experiment: dynamic services", 89, [ surface("experiment:dynamic-services", "playwright", "test/runner-fixtures/dynamic-services/playwright.config.ts", { kind: "experiment", diff --git a/test/repo/blaxel-pool-discovery.test.ts b/test/repo/blaxel-pool-discovery.test.ts index 7d546e46..591cc3ee 100644 --- a/test/repo/blaxel-pool-discovery.test.ts +++ b/test/repo/blaxel-pool-discovery.test.ts @@ -40,7 +40,7 @@ async function withApi(run: (discover: (deployedOnly: boolean) => Promise Promise Promise { await withApi(async (discover, requests) => { - expect(await discover(deployedOnly)).toEqual(deployedOnly ? ["a-worker", "b-worker"] : ["a-worker", "b-worker", "c-failed"]); + expect(await discover(deployedOnly)).toEqual(deployedOnly ? ["a-worker", "b-worker"] : ["a-worker", "b-worker", "c-failed", "d-provisioning", "e-unavailable"]); expect(requests).toEqual(["/sandboxes", "/sandboxes?cursor=second", "/sandboxes?cursor=third"]); }); }); diff --git a/test/repo/blaxel-worker-lease.test.ts b/test/repo/blaxel-worker-lease.test.ts index 007c32a8..dccb84e2 100644 --- a/test/repo/blaxel-worker-lease.test.ts +++ b/test/repo/blaxel-worker-lease.test.ts @@ -1,9 +1,10 @@ -import { spawnSync } from "node:child_process"; -import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawn, spawnSync, execFile } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync, existsSync } from "node:fs"; import os from "node:os"; import path from "node:path"; +import { promisify } from "node:util"; import { afterEach, expect, test } from "vitest"; -import { workerLeaseCommand } from "../../scripts/test/blaxel-worker-lease.mjs"; +import { workerLeaseCommand, workerEntryCommand, shardScratchPath } from "../../scripts/test/blaxel-worker-lease.mjs"; // Real generated lock command, Linux flock/procfs and disposable files. The // fixture-token filter substitutes only the whole-worker audit's scope. @@ -11,7 +12,7 @@ const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); function fixture() { const tmpRoot = mkdtempSync(path.join(os.tmpdir(), "lease-seam-")); roots.push(tmpRoot); - const input = { lockId: "fixture-lock", runId: "fixture-run", shardIndex: 1, shardTotal: 1, attemptId: "fixture-attempt", owner: "fixture", expiresAt: Math.floor(Date.now() / 1000) + 1200, tmpRoot, resultsRoot: path.join(tmpRoot, "results"), auditTokens: ["fixture-lock"] }; + const input = { lockId: "fixture-lock", runId: "fixture-run", shardIndex: 1, shardTotal: 1, attemptId: "fixture-attempt", owner: "fixture", expiresAt: Math.floor(Date.now() / 1000) + 1200, tmpRoot, resultsRoot: path.join(tmpRoot, "results"), auditTokens: ["fixture-lock", "unrelated-lock", "new-lock"] }; const recordPath = path.join(tmpRoot, "tv-testshard.lock", "owner.json"); const run = (extra = {}) => spawnSync("bash", ["-c", workerLeaseCommand({ ...input, ...extra })], { encoding: "utf8", env: process.env, timeout: 15_000 }); return { input, recordPath, run, record: () => JSON.parse(readFileSync(recordPath, "utf8")) }; @@ -50,3 +51,77 @@ test.skipIf(process.platform !== "linux")("renewal and release reject old identi expect(f.run({ operation: "renew" }).status).toBe(42); expect(f.run({ operation: "release", retrieved: true }).status).toBe(42); }); + +const exec = promisify(execFile); +async function until(predicate: () => boolean, timeout = 5000) { + const deadline = Date.now() + timeout; + while (!predicate()) { if (Date.now() > deadline) throw Error("fixture rendezvous timeout"); await new Promise(resolve => setTimeout(resolve, 25)); } +} +function shell(value: string) { return `'${value.replaceAll("'", `'"'"'`)}'`; } + +test.skipIf(process.platform !== "linux")("competing acquisitions serialize, and killing a guard holder releases the OS guard", async () => { + const f = fixture(); + const command = (lockId: string) => exec("bash", ["-c", workerLeaseCommand({ ...f.input, lockId })], { env: process.env, timeout: 10_000 }); + const outcomes = await Promise.allSettled([command("fixture-lock"), command("new-lock")]); + expect(outcomes.filter(result => result.status === "fulfilled")).toHaveLength(1); + const owner = f.record().lockId; + rmSync(path.dirname(f.recordPath), { recursive: true }); + const ready = path.join(f.input.tmpRoot, "guard-ready"); + const child = spawn("flock", ["-o", path.join(f.input.tmpRoot, "tv-testshard.guard"), process.execPath, "-e", `require('fs').writeFileSync(${JSON.stringify(ready)}, 'ready');setInterval(()=>{},1000)`], { detached: true, stdio: "ignore", env: process.env }); + try { + await until(() => existsSync(ready)); + child.kill("SIGKILL"); + await command(owner); + expect(f.record().lockId).toBe(owner); + } finally { try { process.kill(-child.pid!, "SIGKILL"); } catch {} } +}); + +test.skipIf(process.platform !== "linux")("an obsolete delayed entry cannot touch checkout after recovery", async () => { + const f = fixture(); const processName = "delayed"; + expect(f.run().status).toBe(0); expect(f.run({ operation: "prepare" }).status).toBe(0); + const script = path.join(shardScratchPath(f.input), "worker.sh"); + const sentinel = path.join(f.input.tmpRoot, "checkout-touched"); + writeFileSync(script, `touch ${shell(sentinel)}\n`); + expect(f.run({ operation: "dispatch", processName }).status).toBe(0); + const entry = workerEntryCommand({ ...f.input, processName }, script); + // The real entry command is delayed until after the old identity is replaced. + writeFileSync(f.recordPath, JSON.stringify({ ...f.record(), recoverAfter: 1 })); + expect(f.run({ lockId: "new-lock", attemptId: "new-attempt" }).status).toBe(0); + const delayed = spawnSync("bash", ["-c", entry], { encoding: "utf8", timeout: 10_000 }); + expect(delayed.status).not.toBe(0); expect(existsSync(sentinel)).toBe(false); + expect(f.record().lockId).toBe("new-lock"); +}); + +test.skipIf(process.platform !== "linux")("recovery kills registered descendants by token, including TERM-resistant children, and holds for another owner", async () => { + const f = fixture(); const processName = "live-fixture"; + expect(f.run().status).toBe(0); expect(f.run({ operation: "prepare" }).status).toBe(0); + const script = path.join(shardScratchPath(f.input), "worker.sh"); + const ready = path.join(f.input.tmpRoot, "child-ready.json"); + const childSource = `process.on('SIGTERM',()=>{});require('fs').writeFileSync(${JSON.stringify(ready)},JSON.stringify({pid:process.pid,outer:process.env.TV_TEST_SHARD_OWNER,surface:process.env.TV_TEST_SURFACE_OWNER}));setInterval(()=>{},1000)`; + // A real nested SurfaceSupervisor replaces the surface token but preserves the + // outer attempt, even with a restricted requested child environment. + const supervisor = path.join(process.cwd(), "scripts/test/surface-supervisor.mjs"); + const parentSource = `import {SurfaceSupervisor} from ${JSON.stringify(supervisor)};const s=new SurfaceSupervisor({runId:'nested-fixture',surfaceIds:['nested']});await s.run(process.execPath,['-e',${JSON.stringify(childSource)}],{env:{PATH:process.env.PATH}});`; + writeFileSync(script, `exec ${shell(process.execPath)} --input-type=module -e ${shell(parentSource)}\n`); + expect(f.run({ operation: "dispatch", processName }).status).toBe(0); + const child = spawn("bash", ["-c", workerEntryCommand({ ...f.input, processName }, script)], { detached: true, stdio: "ignore", env: process.env }); + const unrelated = spawn(process.execPath, ["-e", "setInterval(()=>{},1000)"], { detached: true, stdio: "ignore", env: { ...process.env, TV_TEST_SHARD_OWNER: "unrelated-lock" } }); + try { + await until(() => existsSync(ready)); + const observed = JSON.parse(readFileSync(ready, "utf8")); + expect(observed.outer).toBe(f.input.lockId); expect(observed.surface).toMatch(/^tv1\./); + expect(f.record().dispatch.pid).toBeGreaterThan(0); + writeFileSync(f.recordPath, JSON.stringify({ ...f.record(), recoverAfter: 1 })); + const held = f.run({ lockId: "new-lock", attemptId: "new-attempt" }); + expect(held.status, held.stderr).toBe(42); + expect(() => process.kill(unrelated.pid!, 0)).not.toThrow(); + const stat = existsSync(`/proc/${observed.pid}/stat`) ? readFileSync(`/proc/${observed.pid}/stat`, "utf8") : ""; + expect(stat === "" || stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z")).toBe(true); + unrelated.kill("SIGKILL"); await until(() => unrelated.signalCode !== null); + const acquired = f.run({ lockId: "new-lock", attemptId: "new-attempt" }); + expect(acquired.status, acquired.stderr).toBe(0); + } finally { + try { process.kill(-child.pid!, "SIGKILL"); } catch {} + try { process.kill(-unrelated.pid!, "SIGKILL"); } catch {} + } +}, 20_000); diff --git a/test/repo/test-runner-config.test.ts b/test/repo/test-runner-config.test.ts index b2d76a1e..73dcc8a3 100644 --- a/test/repo/test-runner-config.test.ts +++ b/test/repo/test-runner-config.test.ts @@ -242,3 +242,15 @@ function visit(dir: string, root: string, out: Set): void { else if (/\.(test|spec)\.ts$/.test(entry.name)) out.add(path.relative(root, full)); } } + +// Source contract; the private driver carries real planned/targeted transport. +test("keeps Blaxel harness fixtures separate from the driver and ordinary suites", () => { + const registry = loadTestConfig(); + for (const suite of ["all", "e2e-ci"]) expect(selectSurfaces(registry, { suite }).some(surface => surface.id.startsWith("experiment:blaxel-harness"))).toBe(false); + expect(selectSurfaces(registry, { tag: "blaxel-harness-fixture" }).map(surface => surface.id)).toEqual(["experiment:blaxel-harness-fixtures"]); + for (const surface of registry.surfaces.filter(surface => surface.id.startsWith("experiment:blaxel-harness"))) expect(surface.kind).toBe("experiment"); + const source = readFileSync("test/runner-fixtures/blaxel-harness/driver/harness.test.ts", "utf8"); + expect(source).toContain('["--tag", "blaxel-harness-fixture", "--shards", "2"'); + expect(source).toContain('["--surface", "experiment:blaxel-harness-fixtures"]'); + expect(source).not.toContain('["--surface", "experiment:blaxel-harness"]'); +}); diff --git a/test/repo/test-runner-provider.test.ts b/test/repo/test-runner-provider.test.ts index 8bf3c27e..4921d443 100644 --- a/test/repo/test-runner-provider.test.ts +++ b/test/repo/test-runner-provider.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { describe, expect, test } from "vitest"; import { combineAttemptReports, compactProviderSummary, failedTasksFromShards, formatShardLogs, infrastructureRetryShardIndices, providerCommandExitCode, readShardSummaries, runInfrastructureAttempts, summarizeInfrastructureAttempts, writeProviderReports, type AttemptReport } from "../../scripts/test/provider-outputs.mjs"; import { classifyBlaxelCoordinatorFailure, classifyBlaxelShardStatus, classifyDownloadedShardStatus, validateDownloadedShardSummary } from "../../scripts/test/blaxel-classification.mjs"; -import { acquireShardLeases, excludeSandboxesByName, leaseAcquisitionFailureShards } from "../../scripts/test/blaxel-lease-acquisition.mjs"; +import { acquireShardLeases, excludeSandboxesByName, leaseAcquisitionFailureShards, leaseRecoveryArgs } from "../../scripts/test/blaxel-lease-acquisition.mjs"; import { normalizeBlaxelSurfaces, normalizeProviderShardSurfaces, normalizeProviderTimingShards, resolveProviderRunGit } from "../../scripts/test/provider-normalization.mjs"; import { loadTestConfig, selectSurfaces } from "../../scripts/test/config.mjs"; import { normalizeProcessLeak } from "../../scripts/test/timing-events.mjs"; @@ -27,6 +27,12 @@ function writeVitestNative(file: string, total: number) { } describe("provider result normalization", () => { + test("forwards only an explicitly supplied recovery interval to the coordinator", () => { + expect(leaseRecoveryArgs({})).toEqual([]); + expect(leaseRecoveryArgs({ "lock-ttl-seconds": "60" })).toEqual(["--lock-ttl-seconds", "60"]); + expect(readFileSync("scripts/test/cli.mjs", "utf8")).toContain("args.push(...leaseRecoveryArgs(selectionOptions))"); + }); + test("constructs Blaxel restore/wake timing with a typed passed status", () => { const source = readFileSync("scripts/run-blaxel-testshards.mjs", "utf8"); expect(source).toContain('restorePhase = { name: "sandbox-restore-wake", status: "passed"'); @@ -55,16 +61,13 @@ describe("provider result normalization", () => { expect(source).toContain("Delete the pool before ensure so every provisioned worker is fresh."); expect(source.indexOf("const existing = await listPoolSandboxes()")) .toBeLessThan(source.indexOf("SandboxInstance.createIfNotExists")); - expect(source.indexOf("await sandbox.wait()")) - .toBeLessThan(source.indexOf("await provisionSandboxRuntime(sandbox)")); - expect(source.indexOf("await provisionSandboxRuntime(sandbox)")) - .toBeLessThan(source.indexOf("await verifySandboxArchitecture(sandbox)")); + expect(source).toContain("await publishWorker(sandbox"); expect(source).toContain("options.size ?? String(DEFAULT_BLAXEL_POOL_SIZE)"); expect(source).toContain('const memory = Number.parseInt(options.memory ?? "8192", 10)'); expect(source).toContain('const image = options.image ?? "blaxel/playwright-chromium:latest"'); expect(source).toContain('return `tv-testshard-${arch}-${pool}-${String(index).padStart(2, "0")}`'); - for (const label of ['project: "television"', 'purpose: "testshards"', "pool,", "arch,"]) { + for (const label of ['project: "television"', 'purpose: "testshards-provisioning"', "pool,", "arch,"]) { expect(source).toContain(label); } }); @@ -333,30 +336,19 @@ describe("provider result normalization", () => { expect(source).toContain("MAX_PARALLEL_LEASE_ACQUISITIONS = 32"); expect(source).toContain("await acquireShardLeases({"); expect(source).toContain("acquire: async ({ shardIndex, slot }) => {"); - expect(source).toContain("const lease = await acquireAny(sandboxes, { shardIndex, slot, requestedWorkers: shardIndices.length });"); - expect(source).toContain("const lockId = `${runId}-shard-${shardIndex}`"); + expect(source).toContain("const lease = await acquireAny(sandboxes, { shardIndex, slot, requestedWorkers: shardIndices.length, attempt: attempt + 1 });"); + expect(source).toContain("const lockId = `${runId}-s${shardIndex}-a${attempt}-"); expect(source).toContain("activeLeases.set(lockId, lease)"); expect(source).toContain("lease.executionRequested = true"); expect(source).toContain("lease.executionCompletionConfirmed = true"); expect(source).toContain("killProcess: lease.executionRequested === true && lease.executionCompletionConfirmed !== true"); expect(source).not.toContain("acquired.map((lease) => stopAndReleaseLease(lease, { killProcess: false }))"); expect(source).toContain("lease.logPath = logPath"); - const stopIndex = source.indexOf("await lease.sandbox.process.kill(lease.processName)"); - const ownerCleanupIndex = source.indexOf("await reapUnconfirmedShardOwners(lease)", stopIndex); - const releaseIndex = source.indexOf("await releaseLock(lease.sandbox, lease.lockId)", ownerCleanupIndex); - expect(stopIndex).toBeGreaterThan(0); - expect(ownerCleanupIndex).toBeGreaterThan(stopIndex); - expect(releaseIndex).toBeGreaterThan(ownerCleanupIndex); - expect(source).toContain("node scripts/test/stale-owner-reaper-cli.mjs --output"); - const ownerCleanupStart = source.indexOf("async function reapUnconfirmedShardOwners"); - const ownerCleanupEnd = source.indexOf("function buildInterruptedReportSnapshot", ownerCleanupStart); - const ownerCleanupSource = source.slice(ownerCleanupStart, ownerCleanupEnd); - expect(ownerCleanupSource).toContain('"set -eo pipefail"'); - expect(ownerCleanupSource).not.toContain('"set -euo pipefail"'); - expect(ownerCleanupSource).toContain('"nvm use --silent default"'); - expect(ownerCleanupSource).not.toContain(' "nvm use --silent",'); + expect(source).toContain("await finishLease({"); + expect(source).toContain('teardown: () => confirmTeardown(lease)'); + expect(source).not.toContain("release_lock()"); + const ownerCleanupSource = source.slice(source.indexOf("async function leaseOperation"), source.indexOf("async function acquireAny")); expect(ownerCleanupSource.indexOf('. "$NVM_DIR/nvm.sh"')).toBeLessThan(ownerCleanupSource.indexOf('"set -u"')); - expect(ownerCleanupSource.indexOf('"set -u"')).toBeLessThan(ownerCleanupSource.indexOf("node scripts/test/stale-owner-reaper-cli.mjs --output")); const nvmFixture = mkdtempSync(path.join(os.tmpdir(), "tv-provider-nvm-nounset-")); const outsideRepository = path.join(nvmFixture, "outside-repository"); @@ -401,10 +393,7 @@ describe("provider result normalization", () => { rmSync(nvmFixture, { recursive: true, force: true }); } - expect(source).toContain("report.cleanupConfirmed === true"); - expect(source).toContain("report.liveOwners.length === 0"); - expect(source).toContain("report.remainingStaleOwners.length === 0"); - expect(source).toContain("Owner cleanup was not confirmed; retaining lease"); + }); test("plans before Blaxel leases and invokes the production explicit-file worker", () => { diff --git a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts new file mode 100644 index 00000000..1fec7560 --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts @@ -0,0 +1,173 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { SandboxInstance } from "@blaxel/core"; +import { expect, test } from "vitest"; +import { listBlaxelPoolSandboxes } from "../../../../scripts/test/blaxel-pool.mjs"; + +// Acceptance: real canonical subprocesses, SDK, fresh private workers and native +// fixtures. The sole scheduling hook pauses immediately before report download. +const pool = "rose-cleanup"; +const coordinatorName = "tv-cleanup-coordinator-rose"; +const planned = ["--tag", "blaxel-harness-fixture", "--shards", "2", "--allow-shard-count-override", "1"]; +const targeted = ["--surface", "experiment:blaxel-harness-fixtures"]; +const resultsRoot = "/workspace/television/.testshards/results"; +const modePath = "/tmp/rose-blaxel-harness-mode.json"; +const cli = path.resolve("scripts/test/cli.mjs"); +const children = new Set(); +const quote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`; +const pause = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); +async function until(probe: () => Promise, timeout = 120_000): Promise { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { const result = await probe(); if (result) return result; await pause(250); } + throw Error("live harness rendezvous timed out"); +} +function readJson(file: string) { return JSON.parse(readFileSync(file, "utf8")); } +async function remoteJson(worker: SandboxInstance, file: string) { try { return JSON.parse(await worker.fs.read(file)); } catch (error: any) { if (error?.status === 404) return null; throw error; } } + +function command(root: string, name: string, args: string[], env: NodeJS.ProcessEnv = {}) { + const log = path.join(root, `${name}.log`); + const child = spawn(process.execPath, [cli, ...args], { cwd: process.cwd(), env: {...process.env, ...env}, stdio: ["ignore", "pipe", "pipe"] }); + children.add(child); + const chunks: Buffer[] = []; + child.stdout!.on("data", data => chunks.push(data)); child.stderr!.on("data", data => chunks.push(data)); + const done = new Promise((resolve, reject) => { + child.on("error", reject); + child.on("close", code => { children.delete(child); writeFileSync(log, Buffer.concat(chunks)); resolve(code); }); + }); + return {child, done, log}; +} +function run(root: string, name: string, selection: string[], gate?: string) { + const pointer = path.join(root, `${name}.run`); + const task = command(root, name, ["blaxel", "--pool", pool, ...selection, "--lock-ttl-seconds", "60", "--run-dir-output", pointer], gate ? {TV_TEST_RUNNER_SELFTEST: "1", TV_BLAXEL_DOWNLOAD_GATE: gate} : {}); + return {...task, directory: async () => until(async () => existsSync(pointer) ? readFileSync(pointer, "utf8").trim() : null)}; +} +async function poolCommand(root: string, name: string, operation: string) { + return command(root, name, ["pool", "blaxel", operation, "--pool", pool, ...(operation === "ensure" ? ["--size", "2"] : [])]).done; +} +async function gates(gate: string) { + return until(async () => [1,2].every(index => existsSync(path.join(gate, `shard-${index}.json`))) ? [1,2].map(index => readJson(path.join(gate, `shard-${index}.json`))) : null); +} +function retained(runDir: string) { + const summary = readJson(path.join(runDir, "summary.json")); + const provider = readJson(path.join(runDir, "provider/blaxel/report.json")); + const normalized = readJson(path.join(runDir, "results.json")); + return { summary, provider, normalized }; +} +function assertDiagnostics(runDir: string, expectedRecovery = false) { + const {provider, normalized} = retained(runDir); + const sidecars = normalized.surfaces.flatMap((surface: any) => surface.resourceDiagnosticsPaths ?? []) as string[]; + expect(sidecars.length).toBeGreaterThan(0); + const records = sidecars.map(file => readJson(path.resolve(runDir, file))); + for (const record of records) { + expect(record.identity.runId).toBe(provider.runId); + expect(record.identity.attemptId).toContain(provider.runId); + if (record.kind === "surface") { + expect(record.start).toBeTruthy(); expect(record.end).toBeTruthy(); + expect(["available", "unavailable"]).toContain(record.kernel?.status ?? "unavailable"); + } + } + if (expectedRecovery) { + const receipts = records.filter(record => record.kind === "recovery"); + expect(receipts.length).toBe(2); + for (const receipt of receipts) { expect(receipt.discardedReports).toBe(true); expect(receipt.cleanup).toBe("confirmed"); expect(receipt.previous.runId).not.toBe(provider.runId); } + } + return records; +} + +test("qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle", async () => { + if (process.env.TV_BLAXEL_HARNESS_COORDINATOR !== coordinatorName || process.env.TV_TEST_SHARD_OWNER) throw Error("Run only on the dedicated Blaxel coordinator, outside every leased worker"); + const coordinator = await SandboxInstance.get(coordinatorName); + expect(coordinator.metadata.labels?.purpose).not.toBe("testshards"); + const root = mkdtempSync(path.resolve(".test-runs/blaxel-live-")); + writeFileSync(path.join(root, "identity.json"), JSON.stringify({coordinatorName, pool, revision: readFileSync(".git/HEAD", "utf8").trim()})); + let workers: SandboxInstance[] = []; + try { + // The caller retained earlier development reports before this fresh-pool run. + expect(await poolCommand(root, "initial-delete", "delete")).toBe(0); + expect(await poolCommand(root, "ensure", "ensure")).toBe(0); + workers = await listBlaxelPoolSandboxes({pool, arch: "x64", deployedOnly: true}); + expect(workers).toHaveLength(2); + for (const worker of workers) { + const setup = "mkdir -p /tmp/television-source-cli-e2e-harness /tmp/television-web-bundle-harness /tmp/tv-testshards/stale-harness /tmp/rose-harness-unrelated /cache/rose-harness; printf fixture > /tmp/television-source-cli-e2e-harness/data; printf fixture > /tmp/television-web-bundle-harness/data; printf keep > /tmp/rose-harness-unrelated/keep; printf keep > /cache/rose-harness/keep"; + const seeded = await worker.process.exec({name: "rose-seed-owned-fixtures", command: `bash -c ${quote(setup)}`, waitForCompletion: true, timeout: 10}); + expect(seeded.exitCode).toBe(0); + await worker.fs.write(modePath, JSON.stringify({mode: "normal"})); + } + const gate = path.join(root, "normal-download"); mkdirSync(gate); + const first = run(root, "planned", planned, gate); + const waiting = await gates(gate); + const before = await Promise.all(workers.map(worker => remoteJson(worker, "/tmp/tv-testshard.lock/owner.json"))); + await until(async () => { + const after = await Promise.all(workers.map(worker => remoteJson(worker, "/tmp/tv-testshard.lock/owner.json"))); + return after.every((record, index) => record.lastRenewal > before[index].lastRenewal) ? after : null; + }); + const competed = run(root, "competing", planned); + expect(await competed.done).not.toBe(0); + expect(retained(await competed.directory()).summary.run.infraStatus).toBe("incomplete"); + expect(await poolCommand(root, "held-delete", "delete")).toBe(1); + for (const worker of workers) expect((await remoteJson(worker, "/tmp/tv-testshard.lock/owner.json")).runId).toBe(before[0].runId); + writeFileSync(path.join(gate, "continue"), "continue"); + expect(await first.done).toBe(1); // intentional native fixture failure + const firstDir = await first.directory(); const firstResult = retained(firstDir); + expect(firstResult.summary.run.infraStatus).toBe("completed"); + expect(firstResult.summary.counts.testsFailed).toBe(1); expect(firstResult.summary.counts.testsFlakyRecovered).toBe(1); + assertDiagnostics(firstDir); + for (const shard of firstResult.provider.shards) { + expect(shard.remoteScript).toBe(`${shard.scratch}/worker.sh`); + expect(shard.setupTimings.some((phase: any) => phase.name === "temporary-cleanup")).toBe(true); + const artifact = path.join(firstDir, "provider/blaxel/artifacts", `shard-${shard.shardIndex}`); + const cleanup = readJson(path.join(artifact, "temporary-cleanup.json")); + expect(cleanup.removed["television-source-cli-e2e-"].directories).toBe(1); + expect(cleanup.removed["television-web-bundle-"].directories).toBe(1); + expect(cleanup.removed["stale-scratch"].directories).toBe(1); + const placement = path.join(artifact, "temporary-placement.json"); + if (existsSync(placement)) expect(readJson(placement).descriptors.some((entry: any) => entry.path.endsWith(" (deleted)"))).toBe(true); + } + for (const worker of workers) { + const ownGate = waiting.find(entry => entry.sandbox === worker.metadata.name)!; + expect(await remoteJson(worker, "/tmp/tv-testshard.lock/owner.json")).toBeNull(); + await expect(worker.fs.ls(ownGate.scratch)).rejects.toMatchObject({status: 404}); + expect(await worker.fs.read("/tmp/rose-harness-unrelated/keep")).toBe("keep"); + expect(await worker.fs.read("/cache/rose-harness/keep")).toBe("keep"); + } + const target = run(root, "targeted", targeted); + expect(await target.done).toBe(1); + expect(retained(await target.directory()).summary.run.infraStatus).toBe("completed"); + assertDiagnostics(await target.directory()); + + for (const scenario of ["active", "quiet"]) { + for (const worker of workers) await worker.fs.write(modePath, JSON.stringify({mode: scenario === "active" ? "active" : "normal"})); + const crashGate = path.join(root, `${scenario}-download`); mkdirSync(crashGate); + const abandoned = run(root, `${scenario}-abandoned`, planned, crashGate); + let owners: any[]; + if (scenario === "quiet") { await gates(crashGate); owners = await Promise.all(workers.map(worker => remoteJson(worker, "/tmp/tv-testshard.lock/owner.json"))); } + else owners = await until(async () => { + const active = await Promise.all(workers.map(worker => remoteJson(worker, `${resultsRoot}/active-fixture.json`))); + return active.every(Boolean) ? Promise.all(workers.map(worker => remoteJson(worker, "/tmp/tv-testshard.lock/owner.json"))) : null; + }); + const coordinatorPid = Number(owners[0].owner.split(":").at(-1)); + expect(coordinatorPid).toBeGreaterThan(1); expect(coordinatorPid).not.toBe(process.pid); + process.kill(coordinatorPid, "SIGKILL"); // actual renewer, not just its shard + expect(await abandoned.done).not.toBe(0); + for (const worker of workers) await worker.fs.write(modePath, JSON.stringify({mode: "normal"})); + await until(async () => Date.now() > Math.max(...owners.map(record => record.recoverAfter)) + 500, 90_000); + const recovered = run(root, `${scenario}-recovered`, planned); + expect(await recovered.done).toBe(1); + const directory = await recovered.directory(); + expect(retained(directory).summary.run.infraStatus).toBe("completed"); + const records = assertDiagnostics(directory, true); + if (scenario === "active") expect(records.filter(record => record.kind === "recovery").every(record => record.processes.length > 0)).toBe(true); + writeFileSync(path.join(root, `${scenario}-recovery-evidence.json`), JSON.stringify({owners, recoveredRun: directory})); + } + expect(await poolCommand(root, "final-delete", "delete")).toBe(0); + expect(await listBlaxelPoolSandboxes({pool, arch: "x64"})).toHaveLength(0); + const absent = run(root, "absent-pool", targeted); expect(await absent.done).not.toBe(0); + writeFileSync(path.join(root, "completed.json"), JSON.stringify({status: "passed", runs: readdirSync(root).filter(file => file.endsWith(".run"))})); + } finally { + for (const child of children) child.kill("SIGTERM"); + // Ordinary manager refusal is retained; never bypass a live lease to tidy up. + const cleanup = await poolCommand(root, "finally-delete", "delete"); + writeFileSync(path.join(root, "cleanup.json"), JSON.stringify({exitCode: cleanup})); + } +}); diff --git a/test/runner-fixtures/blaxel-harness/driver/vitest.config.ts b/test/runner-fixtures/blaxel-harness/driver/vitest.config.ts new file mode 100644 index 00000000..290cbc8b --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/driver/vitest.config.ts @@ -0,0 +1,2 @@ +import { defineConfig } from "vitest/config"; +export default defineConfig({ root: import.meta.dirname, test: { include: ["*.test.ts"], maxWorkers: 1, fileParallelism: false, testTimeout: 1_200_000, hookTimeout: 120_000 } }); diff --git a/test/runner-fixtures/blaxel-harness/fixtures/outcomes.test.ts b/test/runner-fixtures/blaxel-harness/fixtures/outcomes.test.ts new file mode 100644 index 00000000..39b8894d --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/fixtures/outcomes.test.ts @@ -0,0 +1,7 @@ +import { expect, test } from "vitest"; +import { activeFixture, mode } from "./support.ts"; + +test("bounded active owned process", activeFixture); +let attempt = 0; +test("recovers a deliberate first attempt", () => { if (mode() === "normal") expect(++attempt).toBeGreaterThan(1); }); +test("retains a deliberate native failure", () => { if (mode() === "normal") expect("fixture failure").toBe("fixture success"); }); diff --git a/test/runner-fixtures/blaxel-harness/fixtures/support.ts b/test/runner-fixtures/blaxel-harness/fixtures/support.ts new file mode 100644 index 00000000..8a622e16 --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/fixtures/support.ts @@ -0,0 +1,20 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { spawnOwnedProcess } from "../../../helpers/owned-process.ts"; + +export const results = "/workspace/television/.testshards/results"; +export function mode() { + const file = "/tmp/rose-blaxel-harness-mode.json"; + return existsSync(file) ? JSON.parse(readFileSync(file, "utf8")).mode : "normal"; +} +export function evidence(name: string, value: unknown) { + mkdirSync(results, { recursive: true }); + writeFileSync(path.join(results, name), JSON.stringify(value)); +} +export async function activeFixture() { + if (mode() !== "active") return; + const child = spawnOwnedProcess(process.execPath, ["-e", "process.on('SIGTERM',()=>{});setTimeout(()=>process.exit(0),180000);setInterval(()=>{},1000)"], { stdio: "ignore" }); + evidence("active-fixture.json", { pid: child.pid, supervisor: process.pid, token: process.env.TV_TEST_SHARD_OWNER }); + try { await new Promise(resolve => setTimeout(resolve, 180_000)); } + finally { await child.dispose(); } +} diff --git a/test/runner-fixtures/blaxel-harness/fixtures/temporary.test.ts b/test/runner-fixtures/blaxel-harness/fixtures/temporary.test.ts new file mode 100644 index 00000000..78edc8ea --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/fixtures/temporary.test.ts @@ -0,0 +1,33 @@ +import { expect, test } from "vitest"; +import { mkdtempSync, readdirSync, readFileSync, readlinkSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { chromium } from "playwright"; +import { activeFixture, evidence } from "./support.ts"; + +test("bounded active owned process", activeFixture); +test("Node and Chromium place real temporary files under the shard attempt", async () => { + const tmp = os.tmpdir(); + expect(tmp).toMatch(/^\/tmp\/tv-testshards\/[^/]+\/tmp$/); + const nodeDir = mkdtempSync(path.join(tmp, "node-placement-")); + writeFileSync(path.join(nodeDir, "created"), "node temporary file"); + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + await page.setContent(''); + await page.screenshot(); + const descriptors: Array<{pid: number; fd: string; path: string}> = []; + for (const pid of readdirSync("/proc").filter(value => /^\d+$/.test(value))) { + try { + const env = readFileSync(`/proc/${pid}/environ`, "utf8"); + const exe = readlinkSync(`/proc/${pid}/exe`); + if (!env.split("\0").includes(`TV_TEST_SHARD_OWNER=${process.env.TV_TEST_SHARD_OWNER}`) || !/chrome|chromium/.test(exe)) continue; + for (const fd of readdirSync(`/proc/${pid}/fd`)) { + try { const target = readlinkSync(`/proc/${pid}/fd/${fd}`); if (target.startsWith(tmp + "/")) descriptors.push({pid: Number(pid), fd, path: target}); } catch {} + } + } catch {} + } + evidence("temporary-placement.json", { tmp, nodeDir, browserVersion: browser.version(), executable: chromium.executablePath(), token: process.env.TV_TEST_SHARD_OWNER, surface: process.env.TV_TEST_SURFACE_OWNER, descriptors }); + expect(descriptors.some(entry => entry.path.includes("Chromium") && entry.path.endsWith(" (deleted)"))).toBe(true); + } finally { await browser.close(); rmSync(nodeDir, {recursive: true, force: true}); } +}); diff --git a/test/runner-fixtures/blaxel-harness/fixtures/vitest.config.ts b/test/runner-fixtures/blaxel-harness/fixtures/vitest.config.ts new file mode 100644 index 00000000..9676b195 --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/fixtures/vitest.config.ts @@ -0,0 +1,2 @@ +import { defineConfig } from "vitest/config"; +export default defineConfig({ root: import.meta.dirname, test: { include: ["*.test.ts"], maxWorkers: 1, fileParallelism: false, testTimeout: 240_000 } }); From e59caa56da1cc8bced9d585ffb880e3844907503 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:05:48 +0000 Subject: [PATCH 37/64] Serialize shutdown cleanup with outstanding worker operations --- scripts/run-blaxel-testshards.mjs | 42 +++++++++++++++------- test/repo/blaxel-lease-contracts.test.ts | 4 +-- test/repo/blaxel-stale-owner-audit.test.ts | 4 ++- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index f638340b..38540473 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -43,6 +43,7 @@ const timeouts = resolveTimeouts(timeoutProfile, options); const playwrightBrowsersPath = options["playwright-browsers-path"] ?? "/home/playwright/.cache/ms-playwright"; const runId = `${new Date().toISOString().replace(/[:.]/g, "-")}-${process.pid}`; const activeLeases = new Map(); +const pendingAcquisitions = new Set(); const attempts = []; let finalReport = null; let activeAttempt = null; @@ -243,6 +244,7 @@ async function runAttempt({ attempt, shardIndices, excludedSandboxes = [], outpu attemptState.leases.set(outcome.shardIndex, outcome.lease); acquired.push(outcome.lease); } + if (shuttingDown) await shutdownPromise; const acquisitionIncomplete = acquisitionOutcomes.some((outcome) => !outcome.lease); let results; if (acquisitionIncomplete) { @@ -336,7 +338,13 @@ async function leaseOperation(lease, operation, extra = {}) { return response; } -async function acquireAny(sandboxes, { shardIndex, slot, requestedWorkers, verifyArchitecture = true, attempt = 1 }) { +function acquireAny(sandboxes, options) { + const pending = acquireWorker(sandboxes, options); + pendingAcquisitions.add(pending); + return pending.finally(() => pendingAcquisitions.delete(pending)); +} + +async function acquireWorker(sandboxes, { shardIndex, slot, requestedWorkers, verifyArchitecture = true, attempt = 1 }) { const lockId = `${runId}-s${shardIndex}-a${attempt}-${randomUUID().slice(0, 8)}`; const input = { lockId, runId, shardIndex, attempt, attemptId: lockId, owner: `${os.hostname()}:${process.pid}`, ttlSeconds: timeouts.lockTtlSeconds, audit: auditStaleOwners }; input.scratch = shardScratchPath(input); @@ -358,6 +366,7 @@ async function acquireAny(sandboxes, { shardIndex, slot, requestedWorkers, verif }); if (verifyArchitecture && !(await verifySandboxArchitecture(sandbox))) { await stopAndReleaseLease(lease, { killProcess: false }); + activeLeases.delete(lockId); continue; } return lease; @@ -419,7 +428,10 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE const summaryPath = path.join(outputDir, `shard-${shardIndex}.json`); const planRemotePath = shardPlanRaw ? `${lease.input.scratch}/plan.json` : null; const dispatch = await (lease.pendingWrites = transferShardInputs({ - write: (remotePath, contents) => sandbox.fs.write(remotePath, contents), + write: (remotePath, contents) => { + if (lease.lost || shuttingDown) throw lease.lost ?? new Error("coordinator interrupted before upload"); + return sandbox.fs.write(remotePath, contents); + }, planRemotePath, planRaw: shardPlanRaw, remoteScript, @@ -440,6 +452,7 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE try { await leaseOperation(lease, "prepare"); await leaseOperation(lease, "dispatch", { processName }); + if (lease.lost || shuttingDown) throw lease.lost ?? new Error("coordinator interrupted before dispatch"); lease.executionRequested = true; onExecutionRequested(); const proc = await (lease.dispatchRequest = sandbox.process.exec({ name: processName, command, env: { GH_TOKEN: blaxelGithubToken }, timeout: timeouts.workerTimeoutSeconds, keepAlive: true })); @@ -1020,11 +1033,13 @@ exit "$test_status" `; } -async function confirmTeardown(lease) { - if (lease.teardownConfirmed) return; - const result = await leaseOperation(lease, "teardown"); - lease.teardownConfirmed = true; - if (lease.logPath) appendLog(lease.logPath, `\n[coordinator] owner teardown: ${JSON.stringify(result.record.teardown)}\n`); +function confirmTeardown(lease) { + lease.teardownPromise ??= (async () => { + const result = await leaseOperation(lease, "teardown"); + lease.teardownConfirmed = true; + if (lease.logPath) appendLog(lease.logPath, `\n[coordinator] owner teardown: ${JSON.stringify(result.record.teardown)}\n`); + })(); + return lease.teardownPromise; } async function retainAttemptEvidence(lease, directory) { @@ -1050,7 +1065,6 @@ async function retainAttemptEvidence(lease, directory) { for (const name of evidence.files) { if (typeof name !== "string" || name.startsWith("/") || name.split("/").includes("..")) throw new Error("invalid evidence path"); const raw = await lease.sandbox.fs.read(`${root}/${name}`); - if (name.endsWith(".json")) JSON.parse(raw); const target = path.join(directory, "artifacts", `shard-${lease.shardIndex}`, name); fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, raw); } @@ -1077,6 +1091,7 @@ async function stopAndReleaseLease(lease, { killProcess }) { // Even a successful named process wait is not a descendant audit. await confirmTeardown(lease); if (killProcess && lease.workPromise) await lease.workPromise; + if (lease.released) return; if (!lease.executionRequested) lease.retrieved = true; if (!lease.retrieved) throw new Error("results not fully retained; keeping lease for recovery"); await leaseOperation(lease, "release", { retrieved: true }); @@ -1144,9 +1159,10 @@ function installSignalHandlers({ killProcesses }) { process.exit(130); } shuttingDown = true; - const leases = [...activeLeases.values()]; - console.error(`Received ${signal}; preserving the interrupted attempt, then ${killProcesses ? "stopping remote processes and " : ""}releasing ${leases.length} Blaxel lease(s)...`); + console.error(`Received ${signal}; waiting for in-flight acquisitions, then retaining results and cleaning up owned workers...`); shutdownPromise = (async () => { + await Promise.allSettled([...pendingAcquisitions]); + const leases = [...activeLeases.values()]; let interruptedSnapshot = null; const { reportError, releaseError, finalizeError } = await finishInterruptedRun({ preserveReport: () => { @@ -1154,8 +1170,10 @@ function installSignalHandlers({ killProcesses }) { writeInterruptedReportSnapshot(interruptedSnapshot); }, releaseLeases: async () => { - await Promise.allSettled(leases.map((lease) => stopAndReleaseLease(lease, { killProcess: killProcesses }))); - for (const lease of leases) activeLeases.delete(lease.lockId); + const results = await Promise.allSettled(leases.map((lease) => stopAndReleaseLease(lease, { killProcess: killProcesses }))); + for (const lease of leases) if (lease.released) activeLeases.delete(lease.lockId); + const failures = results.flatMap((result, index) => result.status === "rejected" ? [`${leases[index].sandbox.metadata.name}: ${result.reason.message}`] : []); + if (failures.length) throw new Error(failures.join("; ")); }, finalizeReport: () => writeInterruptedReportSnapshot(buildInterruptedReportSnapshot(signal) ?? interruptedSnapshot), }); diff --git a/test/repo/blaxel-lease-contracts.test.ts b/test/repo/blaxel-lease-contracts.test.ts index cdfbb1bb..9226c91f 100644 --- a/test/repo/blaxel-lease-contracts.test.ts +++ b/test/repo/blaxel-lease-contracts.test.ts @@ -25,7 +25,7 @@ test("expired active and quiet leases use the same cleanup; uncertain owners hol let owners = active ? [{ shardToken: "old" }] : []; const result = await workerLeaseOperation(f.input, { now: () => 3000, scanOwners: async () => owners, stopOwners: async (token: string) => { calls.push(token); owners = []; return []; } }); expect(result.ok).toBe(true); expect(calls).toEqual(["old"]); - expect(result.record.recovery.discardedReports).toBe(true); + expect(result.record.recovery?.discardedReports).toBe(true); } const f = fixture(expired); const result = await workerLeaseOperation(f.input, { now: () => 3000, stopOwners: async () => [], scanOwners: async () => [{ shardToken: "unrelated" }] }); @@ -99,7 +99,7 @@ test("retrieval retries after teardown without rerunning work; uncertainty never test("pool eligibility follows verification, and deletion reserves then unpublishes", async () => { const calls: string[] = []; const sandbox = { metadata: { name: "fixture", labels: { purpose: "testshards-provisioning", pool: "private" } }, wait: async () => { calls.push("wait"); } }; - const update = async (_name: string, metadata: { labels: { purpose: string } }) => { calls.push(metadata.labels.purpose); }; + const update = async (_name: string, metadata: { labels: Record }) => { calls.push(metadata.labels.purpose); }; await publishWorker(sandbox, { provision: async () => { calls.push("provision"); }, verify: async () => { calls.push("verify"); }, update }); expect(calls).toEqual(["wait", "provision", "verify", "testshards"]); calls.length = 0; diff --git a/test/repo/blaxel-stale-owner-audit.test.ts b/test/repo/blaxel-stale-owner-audit.test.ts index 5bddc464..8abb11f9 100644 --- a/test/repo/blaxel-stale-owner-audit.test.ts +++ b/test/repo/blaxel-stale-owner-audit.test.ts @@ -82,7 +82,9 @@ describe("Blaxel stale-owner audit", () => { expect(source.indexOf(later), `${later} must follow the audit branch`).toBeGreaterThan(auditBranch); } expect(source).toContain("acquireAny(sandboxes, { shardIndex: 1, slot: 0, requestedWorkers: 1, verifyArchitecture: false })"); - expect(source).toContain("const setup = await runSystemDependencySetup(lease)"); + const auditScript = source.slice(source.indexOf("const remoteScript = `${lease.input.scratch}/audit.sh`"), source.indexOf("const rawLogs =", source.indexOf("async function runStaleOwnerAudit"))); + expect(auditScript.indexOf("systemDependenciesCommand()")).toBeLessThan(auditScript.indexOf("[command, ...args]")); + expect(auditScript).toContain('workerEntryCommand({ ...lease.input, processName }, remoteScript)'); expect(source).toContain("run_step system-deps bash -lc ${shellQuote(systemDependenciesCommand())}"); expect(source).toContain("apt-get install -y --no-install-recommends python3 xauth libgtk-3-0 libxtst6 iproute2"); expect(source).not.toContain("psmisc"); From 12939310a7e65970f7100de5457894cb5efa9fa8 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:07:12 +0000 Subject: [PATCH 38/64] Keep optional diagnostics within their bounded transport --- scripts/run-blaxel-testshards.mjs | 3 +++ scripts/test/blaxel-worker-lease.mjs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index 38540473..95ac307d 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -1063,6 +1063,9 @@ async function retainAttemptEvidence(lease, directory) { } if (evidence.runId !== lease.input.runId || evidence.attemptId !== lease.input.attemptId || !Array.isArray(evidence.files)) throw new Error("invalid retained-evidence inventory"); for (const name of evidence.files) { + // Optional diagnostic reads use their bounded, warning-only transport below. + // They must not make a native result fail or bypass its retention budget. + if (typeof name === "string" && name.startsWith("resources/")) continue; if (typeof name !== "string" || name.startsWith("/") || name.split("/").includes("..")) throw new Error("invalid evidence path"); const raw = await lease.sandbox.fs.read(`${root}/${name}`); const target = path.join(directory, "artifacts", `shard-${lease.shardIndex}`, name); diff --git a/scripts/test/blaxel-worker-lease.mjs b/scripts/test/blaxel-worker-lease.mjs index b9327474..a755ab6e 100644 --- a/scripts/test/blaxel-worker-lease.mjs +++ b/scripts/test/blaxel-worker-lease.mjs @@ -224,7 +224,7 @@ export async function cleanWorkerTemporaryDirectories({ tmpRoot = "/tmp", curren for (const root of [tmpRoot, scratchRoot]) { if (!fs.existsSync(root)) continue; if (lstat(root).isSymbolicLink() || !lstat(root).isDirectory()) throw new Error(`unsafe cleanup root ${root}`); - if (root !== tmpRoot && lstat(root).dev !== lstat(tmpRoot).dev) throw new Error("scratch root crosses a mount boundary"); + if (root !== tmpRoot && (mountPoints.has(root) || lstat(root).dev !== lstat(tmpRoot).dev)) throw new Error("scratch root crosses a mount boundary"); } function allocated(file, device) { const stat = lstat(file); From b5016510773ea6bbd6c8145eb2a3de36cb5abf34 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:10:28 +0000 Subject: [PATCH 39/64] Cover renewal and cleanup of partially provisioned private workers --- scripts/test/blaxel-pool-lifecycle.mjs | 8 +++-- test/repo/blaxel-lease-contracts.test.ts | 36 +++++++++++++++++-- .../blaxel-harness/driver/harness.test.ts | 14 +++++--- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/scripts/test/blaxel-pool-lifecycle.mjs b/scripts/test/blaxel-pool-lifecycle.mjs index 10ced551..0f742989 100644 --- a/scripts/test/blaxel-pool-lifecycle.mjs +++ b/scripts/test/blaxel-pool-lifecycle.mjs @@ -8,7 +8,11 @@ export async function publishWorker(sandbox, { provision, verify, update }) { export async function deleteWorker(sandbox, { reserve, update, remove }) { // A fresh sandbox was never eligible, so no coordinator can own its checkout. - if (sandbox.metadata.labels.purpose !== "testshards-provisioning") await reserve(sandbox); - await update(sandbox.metadata.name, { labels: { ...sandbox.metadata.labels, purpose: "testshards-unavailable" } }); + if (sandbox.metadata.labels.purpose !== "testshards-provisioning") { + await reserve(sandbox); + await update(sandbox.metadata.name, { labels: { ...sandbox.metadata.labels, purpose: "testshards-unavailable" } }); + } + // Keep the never-published state if deletion fails: provisioning may have + // failed before installing the runtime needed by the ownership check. await remove(sandbox.metadata.name); } diff --git a/test/repo/blaxel-lease-contracts.test.ts b/test/repo/blaxel-lease-contracts.test.ts index 9226c91f..2c3f9128 100644 --- a/test/repo/blaxel-lease-contracts.test.ts +++ b/test/repo/blaxel-lease-contracts.test.ts @@ -1,9 +1,9 @@ import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, existsSync, symlinkSync, lstatSync } from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, expect, test } from "vitest"; +import { afterEach, expect, test, vi } from "vitest"; import { workerLeaseOperation, cleanWorkerTemporaryDirectories } from "../../scripts/test/blaxel-worker-lease.mjs"; -import { finishLease } from "../../scripts/test/blaxel-lease-lifecycle.mjs"; +import { finishLease, renewLease } from "../../scripts/test/blaxel-lease-lifecycle.mjs"; import { publishWorker, deleteWorker } from "../../scripts/test/blaxel-pool-lifecycle.mjs"; // Contract clocks, owner audits/signals and SDK operations are substitutions. @@ -114,3 +114,35 @@ test("pool eligibility follows verification, and deletion reserves then unpublis await expect(deleteWorker(sandbox, { ...ops, reserve: async () => { throw Error("valid lease"); } })).rejects.toThrow(); expect(calls).toEqual([]); }); + + +test("renewal continues while retrieval waits and stops on ownership loss", async () => { + vi.useFakeTimers(); + try { + let lost: unknown = null; + const renew = vi.fn().mockResolvedValue(undefined); + const stop = renewLease({ renew, intervalMs: 10, onLost: (error: unknown) => { lost = error; } }); + let continueDownload!: () => void; + const waiting = new Promise(resolve => { continueDownload = resolve; }); + const release = vi.fn(); + const finishing = finishLease({ settleWrites: async () => {}, teardown: async () => {}, download: () => waiting, release, assertOwned: () => { if (lost) throw lost; } }); + await vi.advanceTimersByTimeAsync(35); + expect(renew).toHaveBeenCalledTimes(3); expect(release).not.toHaveBeenCalled(); + continueDownload(); await finishing; await stop(); + await vi.advanceTimersByTimeAsync(30); + expect(renew).toHaveBeenCalledTimes(3); expect(release).toHaveBeenCalledOnce(); + renew.mockRejectedValueOnce(Error("ownership lost")); + const stopLost = renewLease({ renew, intervalMs: 10, onLost: (error: unknown) => { lost = error; } }); + await vi.advanceTimersByTimeAsync(100); + expect(renew).toHaveBeenCalledTimes(4); expect(lost).toBeInstanceOf(Error); + await stopLost(); + } finally { vi.useRealTimers(); } +}); + +test("failed deletion of never-published workers remains retryable without their runtime", async () => { + const sandbox = { metadata: { name: "partial", labels: { purpose: "testshards-provisioning" } } }; + const ops = { reserve: vi.fn().mockRejectedValue(Error("no runtime")), update: vi.fn(), remove: vi.fn().mockRejectedValueOnce(Error("provider unavailable")).mockResolvedValueOnce(undefined) }; + await expect(deleteWorker(sandbox, ops)).rejects.toThrow("provider unavailable"); + await deleteWorker(sandbox, ops); + expect(ops.reserve).not.toHaveBeenCalled(); expect(ops.update).not.toHaveBeenCalled(); expect(ops.remove).toHaveBeenCalledTimes(2); +}); diff --git a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts index 1fec7560..a6069f1c 100644 --- a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts +++ b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts @@ -27,7 +27,7 @@ async function remoteJson(worker: SandboxInstance, file: string) { try { return function command(root: string, name: string, args: string[], env: NodeJS.ProcessEnv = {}) { const log = path.join(root, `${name}.log`); - const child = spawn(process.execPath, [cli, ...args], { cwd: process.cwd(), env: {...process.env, ...env}, stdio: ["ignore", "pipe", "pipe"] }); + const child = spawn(process.execPath, [cli, ...args], { cwd: process.cwd(), detached: true, env: {...process.env, ...env}, stdio: ["ignore", "pipe", "pipe"] }); children.add(child); const chunks: Buffer[] = []; child.stdout!.on("data", data => chunks.push(data)); child.stderr!.on("data", data => chunks.push(data)); @@ -165,9 +165,15 @@ test("qualifies exclusive retrieval, active and quiet recovery, temporary placem const absent = run(root, "absent-pool", targeted); expect(await absent.done).not.toBe(0); writeFileSync(path.join(root, "completed.json"), JSON.stringify({status: "passed", runs: readdirSync(root).filter(file => file.endsWith(".run"))})); } finally { - for (const child of children) child.kill("SIGTERM"); - // Ordinary manager refusal is retained; never bypass a live lease to tidy up. - const cleanup = await poolCommand(root, "finally-delete", "delete"); + const pending = [...children]; + const stopped = Promise.all(pending.map(child => new Promise(resolve => child.once("close", () => resolve())))); + for (const child of pending) { try { process.kill(-child.pid!, "SIGTERM"); } catch {} } + await Promise.race([stopped, pause(10_000)]); + for (const child of children) { try { process.kill(-child.pid!, "SIGKILL"); } catch {} } + // A valid lease is never bypassed. With our coordinators stopped, the same + // manager may delete an expired private worker, which is itself cleanup. + let cleanup = await poolCommand(root, "finally-delete", "delete"); + if (cleanup !== 0) { await pause(65_000); cleanup = await poolCommand(root, "finally-delete-expired", "delete"); } writeFileSync(path.join(root, "cleanup.json"), JSON.stringify({exitCode: cleanup})); } }); From 53d800fa36354019c3d43bbd28f58bbd2b9d2ad4 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:15:52 +0000 Subject: [PATCH 40/64] Preserve attempt evidence across worker checkout reset --- .../test-harness-resource-cleanup-slice-3.md | 73 +++++++++++++++++++ scripts/run-blaxel-testshards.mjs | 7 +- scripts/test/blaxel-worker-lease.mjs | 8 +- test/repo/blaxel-worker-lease.test.ts | 15 ++++ 4 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 docs/working/test-harness-resource-cleanup-slice-3.md diff --git a/docs/working/test-harness-resource-cleanup-slice-3.md b/docs/working/test-harness-resource-cleanup-slice-3.md new file mode 100644 index 00000000..a2b75b23 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-slice-3.md @@ -0,0 +1,73 @@ +# Slice 3: exclusive Blaxel worker lifecycle + +Status: implementation and private qualification in progress; not ready for review. +The converged plan and proofs govern the scope. Slice 2 passed at `296391c0`. +No tests, builds, lint, type checks, browsers or servers run on the shared host. +All executions below use pushed revisions on the private Blaxel coordinator and +`rose-cleanup` workers. No shared-pool worker experiments have been performed. + +## Implementation + +The worker-local lock command carries its implementation before checkout and uses +one short `flock` guard. Its record owns an attempt-specific scratch directory and +an outer process token. Only the coordinator renews; normal release follows owned +process teardown and retained evidence. Expired renewal uses the same cleanup +procedure for active and quiet abandoned work. Old timestamps, incomplete records, +unknown dispatch and unsafe cleanup remain fail-closed at the reviewed boundaries. +Pool publication follows runtime verification; deletion reserves eligible workers +before unpublishing them. A never-published worker keeps that state if deletion +fails, so a partial provisioning failure does not require an installed runtime to +retry deletion. + +The live driver and its two native fixture files are explicit experiment surfaces. +They use only the named private pool, shorten recovery to 60 seconds deliberately, +and retain default native and infrastructure retries. Normal runs retain the +1200-second recovery interval. Qualification remains pending below. + +## Validation record + +| Revision | Command on Blaxel | Retained run / result | +| --- | --- | --- | +| `1320c957` | `npm test -- blaxel --pool rose-cleanup --file test/repo/blaxel-worker-lease.test.ts` | `.test-runs/2026-10-27T18-41-03-525Z-p19298-r7dc71e154247c385`: two intended assertion failures, infrastructure completed. | +| `2c0f5d6b` | Same selected file | `.test-runs/2026-10-27T18-50-52-346Z-p19842-r74788ac655aa50a6`: implementation failure, not green. Vitest rewrote serialized dynamic imports; cleanup also rejected regular overlay files as mount boundaries. | +| `e102e200` | `npm run lint`; `npm run type-check` | Lint passed; type check found contract-test typing errors. `.test-runs/slice-3-static/e102e200.log`. | +| `e59caa56` | Same static commands | Both passed. `.test-runs/slice-3-static/e59caa56.log`. | +| `12939310` | Lease seam, then lease contracts, each selected by file through the canonical Blaxel runner | `.test-runs/2026-10-27T19-11-24-927Z-p21647-r03d2ada7561f5f9a`: all six lease seam cases passed on both workers; canonical retrieval failed because checkout removed `attempt.json`. The run is infrastructure-incomplete, not a green checkpoint. Contracts were not reached. | + +The failed cleanup retained both private leases rather than admitting new work. +Read-only inspection established that the worker overlay exposes device 19 for +directories and 20 for regular files under the same mount. The corrected cleaner +uses actual mount records plus directory device boundaries. It still refuses +mounted paths and symlink roots. Earlier reports and available native files were +copied before the next acquisition can discard abandoned worker evidence. + +The coordinator shutdown path now waits for in-flight acquisitions, shares one +teardown operation with normal completion, and reports failed releases. Optional +resource sidecars use only the bounded warning-only transport; the generic evidence +inventory cannot turn a diagnostic read failure into a native failure or bypass +its byte budget. + +## Review refinements and limits + +Slice 2 refinements 1 and 2 are implemented: reuse the timing-provider constant and +resolve a targeted Vitest config relative to its surface cwd. Recovery receipts +carry the actual prior run/lock, worker, renewal/deadline and attributed process +cleanup, rather than assuming a coordinator's OS death can be proven remotely. +The live receipt, retention and Chromium temporary-file observations remain open. + +Primitive's warning `dec_8e3d38a8` (check +`td7crdek1b1n4dhajyww5p174s8h23bq`) remains unread because its CLI failed to install +or start. The supervisor directed no further debugging and confirmed it is not an +authority over the converged specs and Josh's decisions. Commit-hook scans also +reported partial/truncated results; those are not a complete conflict check. +No new lasting team decision is being established by this implementation. + +The next failed run confirmed expiry recovery, process disuse and scratch deletion +on both private workers. Its retained worker records and native reports are in +`manual-worker-01/` and `manual-worker-02/` under that run directory. The checkout +helper resets `.testshards/results`, so initial identity/cleanup/receipt evidence +must survive in attempt scratch and be restored after checkout, on shell failure, +and by confirmed teardown after a killed shell. The correction adds that fixture +boundary without changing repository selection or checkout behavior. The new +leases remain held until their ordinary deadlines (about 19:31 UTC); no lock was +stolen or deadline edited. diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index 95ac307d..b9b82c32 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -857,8 +857,12 @@ runtime_versions() { node --version npm --version } +restore_attempt_evidence() { + mkdir -p /workspace/television/.testshards/results + cp -R ${shellQuote(`${scratch}/evidence/.`)} /workspace/television/.testshards/results/ +} write_setup_metrics() { - mkdir -p /workspace/television/.testshards/results 2>/dev/null || true + restore_attempt_evidence node - "$setup_metrics" "/workspace/television/.testshards/results/setup-shard-${shardIndex}.json" "/workspace/television/.testshards/results/shard-${shardIndex}.json" "$deps_cache_status" <<'METRICS_NODE' const fs = require('fs'); const [input, setupOutput, shardOutput, depsCacheStatusFile] = process.argv.slice(2); @@ -904,6 +908,7 @@ export GIT_ASKPASS=${scratch}/git-askpass.sh export GIT_TERMINAL_PROMPT=0 run_step checkout bash -lc ${shellQuote(blaxelCheckoutScript({ repoUrl, commit }))} cd /workspace/television +restore_attempt_evidence run_step runtime-activation activate_runtime run_step runtime-versions runtime_versions run_step deps bash -c ${shellQuote(blaxelDependencyScript())} diff --git a/scripts/test/blaxel-worker-lease.mjs b/scripts/test/blaxel-worker-lease.mjs index a755ab6e..78664072 100644 --- a/scripts/test/blaxel-worker-lease.mjs +++ b/scripts/test/blaxel-worker-lease.mjs @@ -124,7 +124,7 @@ export async function workerLeaseOperation(input, hooks = {}) { const processes = await stopOwners(record.lockId); if (!await auditClean()) return denied("other live or stale owners remain"); await clean(null); - recovery = { kind: "recovery", previous: { runId: record.runId, lockId: record.lockId, worker: input.worker ?? "unknown", lastRenewal: record.lastRenewal, recoverAfter: record.recoverAfter }, cleanup: "confirmed", discardedReports: true, processes: processes.slice(0, 128) }; + recovery = { kind: "recovery", previous: { runId: record.runId, lockId: record.lockId, worker: input.worker ?? "unknown", lastRenewal: record.lastRenewal, recoverAfter: record.recoverAfter }, cleanup: "confirmed", discardedReports: true, processes: processes.slice(0, 128), omittedProcesses: Math.max(0, processes.length - 128) }; } else if (!input.audit && !await auditClean()) return denied("live or stale owners remain"); if (operation === "delete") { write({ lockId: input.lockId, deleting: true }); @@ -164,6 +164,9 @@ export async function workerLeaseOperation(input, hooks = {}) { fs.mkdirSync(path.join(resultsRoot, "resources")); fs.writeFileSync(path.join(resultsRoot, "resources/recovery.json"), raw); } + // Checkout can clone/reset the entire repository. Keep this evidence in + // attempt scratch until the shell or teardown restores it after checkout. + fs.cpSync(resultsRoot, path.join(record.scratch, "evidence"), { recursive: true }); record.prepared = true; write(record); } else if (operation === "dispatch") { if (!record.prepared || record.dispatch) return denied("invalid dispatch state"); @@ -181,6 +184,9 @@ export async function workerLeaseOperation(input, hooks = {}) { const processes = await stopOwners(record.lockId); if (!record.audit && !await auditClean()) return denied("worker audit not clean"); if (record.prepared) { + fs.mkdirSync(resultsRoot, { recursive: true }); + const preserved = path.join(record.scratch, "evidence"); + if (fs.existsSync(preserved)) fs.cpSync(preserved, resultsRoot, { recursive: true }); const setupFile = path.join(resultsRoot, `setup-shard-${record.shardIndex}.json`); if (!fs.existsSync(setupFile)) { const inputFile = path.join(record.scratch, "setup.jsonl"); diff --git a/test/repo/blaxel-worker-lease.test.ts b/test/repo/blaxel-worker-lease.test.ts index dccb84e2..fb61934b 100644 --- a/test/repo/blaxel-worker-lease.test.ts +++ b/test/repo/blaxel-worker-lease.test.ts @@ -125,3 +125,18 @@ test.skipIf(process.platform !== "linux")("recovery kills registered descendants try { process.kill(-unrelated.pid!, "SIGKILL"); } catch {} } }, 20_000); + + +test.skipIf(process.platform !== "linux")("attempt identity and recovery evidence survive checkout resetting the result tree", () => { + const f = fixture(); + mkdirSync(path.dirname(f.recordPath)); + writeFileSync(f.recordPath, JSON.stringify({ lockId: "abandoned", runId: "old-run", attemptId: "old-attempt", expiresAt: 0, lastRenewal: 1, recoverAfter: 2 })); + expect(f.run().status).toBe(0); + expect(f.run({ operation: "prepare" }).status).toBe(0); + rmSync(f.input.resultsRoot, { recursive: true }); // checkout's real reset boundary + expect(f.run({ operation: "teardown" }).status).toBe(0); + expect(JSON.parse(readFileSync(path.join(f.input.resultsRoot, "attempt.json"), "utf8"))).toMatchObject({lockId: f.input.lockId, runId: f.input.runId}); + expect(JSON.parse(readFileSync(path.join(f.input.resultsRoot, "resources/recovery.json"), "utf8"))).toMatchObject({previous: {lockId: "abandoned"}, discardedReports: true}); + expect(JSON.parse(readFileSync(path.join(f.input.resultsRoot, "evidence.json"), "utf8")).files).toContain("temporary-cleanup.json"); + expect(existsSync(shardScratchPath(f.input))).toBe(false); +}); From c7482876a86f966f2c72a208aedd663b39991752 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:24:18 +0000 Subject: [PATCH 41/64] Retain recovery evidence and keep the guard on the mutating process --- scripts/manage-blaxel-testshards.mjs | 2 +- scripts/run-blaxel-testshards.mjs | 36 ++++++++++++++----- scripts/test/blaxel-classification.mjs | 7 +++- scripts/test/blaxel-pool-lifecycle.d.mts | 2 +- scripts/test/blaxel-worker-lease.d.mts | 2 +- scripts/test/blaxel-worker-lease.mjs | 11 +++--- scripts/test/resource-diagnostics.mjs | 3 +- test/repo/blaxel-worker-lease.test.ts | 3 +- test/repo/test-runner-provider.test.ts | 9 ++++- .../blaxel-harness/driver/harness.test.ts | 11 +++--- 10 files changed, 61 insertions(+), 25 deletions(-) diff --git a/scripts/manage-blaxel-testshards.mjs b/scripts/manage-blaxel-testshards.mjs index 7c2c348b..4c41f705 100644 --- a/scripts/manage-blaxel-testshards.mjs +++ b/scripts/manage-blaxel-testshards.mjs @@ -36,7 +36,7 @@ if (command === "ensure" && !/^\d+$/.test(nodeSelector)) fail(".nvmrc must conta if (command === "list") { const sandboxes = await listPoolSandboxes(); for (const sandbox of sandboxes) { - console.log(`${sandbox.metadata?.name}\t${sandbox.status}\t${sandbox.spec?.runtime?.memory ?? "?"}MB\t${sandbox.spec?.image ?? "?"}`); + console.log(`${sandbox.metadata?.name}\t${sandbox.status}\t${sandbox.metadata?.labels?.purpose}\t${sandbox.spec?.runtime?.memory ?? "?"}MB\t${sandbox.spec?.image ?? "?"}`); } process.exit(0); } diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index b9b82c32..ebfa1004 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -10,7 +10,7 @@ import { blaxelCheckoutScript, blaxelDependencyScript, resolveBlaxelRepositoryUr import { transferShardInputs } from "./test/blaxel-shard-dispatch.mjs"; import { buildLinuxStaleOwnerAuditInvocation, parseLinuxStaleOwnerAuditOutput } from "./test/blaxel-stale-owner-audit.mjs"; import { normalizeSetupTimingStatus, normalizeTimingPhase } from "./test/phase-metrics.mjs"; -import { RESOURCE_SIDECAR_BYTES, RESOURCE_ATTEMPT_BYTES } from "./test/resource-diagnostics.mjs"; +import { RESOURCE_SIDECAR_BYTES, RESOURCE_ATTEMPT_BYTES, writeResourceDiagnostic } from "./test/resource-diagnostics.mjs"; import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; @@ -346,7 +346,7 @@ function acquireAny(sandboxes, options) { async function acquireWorker(sandboxes, { shardIndex, slot, requestedWorkers, verifyArchitecture = true, attempt = 1 }) { const lockId = `${runId}-s${shardIndex}-a${attempt}-${randomUUID().slice(0, 8)}`; - const input = { lockId, runId, shardIndex, attempt, attemptId: lockId, owner: `${os.hostname()}:${process.pid}`, ttlSeconds: timeouts.lockTtlSeconds, audit: auditStaleOwners }; + const input = { lockId, runId, shardIndex, attempt, attemptId: lockId, owner: `${os.hostname()}:${process.pid}`, ttlSeconds: timeouts.lockTtlSeconds, audit: auditStaleOwners, summaryIdentity: shardSummaryIdentity(shardIndex) }; input.scratch = shardScratchPath(input); for (const sandbox of acquisitionOrder(sandboxes, { slot, requestedWorkers })) { if (sandbox.__leased || shuttingDown) continue; @@ -374,6 +374,17 @@ async function acquireWorker(sandboxes, { shardIndex, slot, requestedWorkers, ve return null; } +function shardSummaryIdentity(shardIndex) { + return { + shardIndex, shardTotal, + ...(shardPlan ? { + planId: shardPlan.planId, timingProvider: shardPlan.timingProvider, + testedCommit: shardPlan.testedCommit, testedTree: shardPlan.testedTree, + assignedFiles: shardPlan.shards[shardIndex - 1].surfaces.flatMap(surface => surface.files.map(file => file.path)).sort(), + } : {}), + }; +} + async function verifySandboxArchitecture(sandbox) { const check = [ "set -euo pipefail", @@ -492,6 +503,7 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE async function downloadResult() { const reportStarted = Date.now(); await leaseOperation(lease, "check"); + resourceDiagnosticsPaths = await copyResourceDiagnostics(sandbox, { outputDir, shardIndex, diagnosticAttemptId, copiedArtifacts }); await retainAttemptEvidence(lease, outputDir); const plannedShard = shardPlan?.shards?.[shardIndex - 1]; let shardSummary = null; @@ -555,11 +567,12 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE } } catch {} retainResultProgress(); - resourceDiagnosticsPaths = await copyResourceDiagnostics(sandbox, { outputDir, shardIndex, diagnosticAttemptId, copiedArtifacts }); - retainResultProgress(); for (const [taskIndex, task] of (shardSummary?.tasks ?? []).entries()) { const copiedTask = { ...tasks[taskIndex] }; - copiedTask.resourceDiagnosticsPaths = (task.resourceDiagnosticsPaths ?? []).map((remotePath) => copiedArtifacts.get(remotePath)).filter(Boolean); + copiedTask.resourceDiagnosticsPaths = [ + ...(task.resourceDiagnosticsPaths ?? []).map((remotePath) => copiedArtifacts.get(remotePath)).filter(Boolean), + ...resourceDiagnosticsPaths.filter((file) => path.basename(file) === "recovery.json"), + ]; for (const field of ["nativeResultPath", "attemptResultPath", "workspaceNativeResultPath", "lifecycleResultPath", "processLeakResultPath"]) { if (!task[field]) continue; copiedTask[field] = await copyRemoteArtifact(sandbox, task[field], { outputDir, shardIndex, copiedArtifacts }); @@ -670,7 +683,11 @@ async function copyResourceDiagnostics(sandbox, { outputDir, shardIndex, diagnos if (record.schemaVersion !== 1 || record.identity?.runId !== runId || record.identity?.shardIndex !== shardIndex || record.identity?.attemptId !== diagnosticAttemptId) throw new Error("identity"); const localPath = path.join(outputDir, "artifacts", `shard-${shardIndex}`, "resources", entry.name); fs.mkdirSync(path.dirname(localPath), { recursive: true }); - fs.writeFileSync(localPath, raw); + if (record.kind === "recovery") { + // The bootstrap receipt must exist before checkout can supply this + // shared writer. Retention adopts its whitelist and attempt budget. + if (!writeResourceDiagnostic(path.dirname(path.dirname(localPath)), path.basename(entry.name, ".json"), record)) continue; + } else fs.writeFileSync(localPath, raw); copiedArtifacts.set(remotePath, localPath); retained.push(localPath); bytes += size; } catch { console.error(`[resource diagnostics] shard ${shardIndex}: sidecar unavailable, invalid or over budget`); } @@ -876,10 +893,13 @@ if (!fs.existsSync(shardOutput)) { const failed = rows.find((row) => row.status && row.status !== 0); if (failed) { const summary = { + ...${JSON.stringify(shardSummaryIdentity(shardIndex))}, status: 'infra-failed', - failureKind: failed.name === 'checkout' ? 'checkout' : 'setup', + collectedFiles: [], + neverProduced: [failed.name === 'test-run' ? 'final shard summary' : 'test execution'], + failureKind: failed.name === 'checkout' ? 'checkout' : failed.name === 'test-run' ? 'worker-execution' : 'setup', failureStep: failed.name, - failureMessage: failed.name === 'checkout' ? 'Repository checkout failed on the Blaxel worker. Check GitHub token validity and repository access from the sandbox.' : 'Setup step ' + failed.name + ' failed before tests ran.', + failureMessage: failed.name === 'checkout' ? 'Repository checkout failed on the Blaxel worker. Check GitHub token validity and repository access from the sandbox.' : failed.name === 'test-run' ? 'Test execution ended without a finalized shard summary.' : 'Setup step ' + failed.name + ' failed before tests ran.', suite: ${JSON.stringify(suite)}, shardIndex: ${shardIndex}, shardTotal: ${shardTotal}, diff --git a/scripts/test/blaxel-classification.mjs b/scripts/test/blaxel-classification.mjs index 87ebbab3..9547c243 100644 --- a/scripts/test/blaxel-classification.mjs +++ b/scripts/test/blaxel-classification.mjs @@ -36,7 +36,12 @@ export function validateDownloadedShardSummary(summary, plan, shardIndex) { const assignedFiles = [...(summary.assignedFiles ?? [])].sort(); const collectedFiles = [...(summary.collectedFiles ?? [])].sort(); if (JSON.stringify(assignedFiles) !== JSON.stringify(expectedFiles)) return `downloaded shard ${shardIndex} assigned files disagree with its plan`; - if (summary.failureKind === "stale-owner" && collectedFiles.length === 0 && (summary.tasks ?? []).length === 0) return null; + if (collectedFiles.length === 0 && (summary.tasks ?? []).length === 0) { + if (summary.failureKind === "stale-owner") return null; + // A setup failure or killed shell can retain its assignment without ever + // producing native results. This remains infrastructure failure, not a pass. + if (summary.status === "infra-failed" && (summary.neverProduced ?? []).some(value => ["test execution", "final shard summary"].includes(value))) return null; + } const taskAssignedFiles = (summary.tasks ?? []).flatMap((task) => task.assignedFiles ?? []).sort(); const taskCollectedFiles = (summary.tasks ?? []).flatMap((task) => task.collectedFiles ?? []).sort(); if (JSON.stringify(taskAssignedFiles) !== JSON.stringify(expectedFiles)) return `downloaded shard ${shardIndex} task assignments disagree with its plan`; diff --git a/scripts/test/blaxel-pool-lifecycle.d.mts b/scripts/test/blaxel-pool-lifecycle.d.mts index 5632ee68..7a4390db 100644 --- a/scripts/test/blaxel-pool-lifecycle.d.mts +++ b/scripts/test/blaxel-pool-lifecycle.d.mts @@ -1,3 +1,3 @@ export interface Worker {metadata: {name: string; labels: Record}; wait: () => Promise} export function publishWorker(worker: Worker, operations: {provision: (worker: Worker) => Promise; verify: (worker: Worker) => Promise; update: (name: string, metadata: {labels: Record}) => Promise}): Promise; -export function deleteWorker(worker: Worker, operations: {reserve: (worker: Worker) => Promise; update: (name: string, metadata: {labels: Record}) => Promise; remove: (name: string) => Promise}): Promise; +export function deleteWorker(worker: Pick, operations: {reserve: (worker: Pick) => Promise; update: (name: string, metadata: {labels: Record}) => Promise; remove: (name: string) => Promise}): Promise; diff --git a/scripts/test/blaxel-worker-lease.d.mts b/scripts/test/blaxel-worker-lease.d.mts index 92762a0c..a9b6cf70 100644 --- a/scripts/test/blaxel-worker-lease.d.mts +++ b/scripts/test/blaxel-worker-lease.d.mts @@ -1,7 +1,7 @@ import type { Stats } from "node:fs"; export const DEFAULT_RECOVERY_SECONDS: number; export const SHARD_OWNER_ENV: string; -export interface LeaseInput { lockId: string; runId?: string; shardIndex?: number; shardTotal?: number; attempt?: number; attemptId?: string; owner?: string; ttlSeconds?: number; expiresAt?: number; tmpRoot?: string; resultsRoot?: string; scratch?: string; auditTokens?: string[]; audit?: boolean; operation?: string; processName?: string; pid?: number; pidFromEnv?: boolean; retrieved?: boolean; worker?: string } +export interface LeaseInput { lockId: string; runId?: string; shardIndex?: number; shardTotal?: number; attempt?: number; attemptId?: string; owner?: string; ttlSeconds?: number; expiresAt?: number; tmpRoot?: string; resultsRoot?: string; scratch?: string; auditTokens?: string[]; audit?: boolean; operation?: string; processName?: string; pid?: number; pidFromEnv?: boolean; retrieved?: boolean; worker?: string; summaryIdentity?: Record } export interface CleanupReport { ok: boolean; before: {availableBytes: number; freeBytes: number}; after: {availableBytes: number; freeBytes: number}; removed: Record; skips: unknown[]; errors: unknown[] } export interface LeaseRecord { lockId: string; runId: string; attemptId: string; recoverAfter: number; lastRenewal: number; expiresAt: number; recovery: {discardedReports: boolean} | null } export function shardScratchPath(input: LeaseInput): string; diff --git a/scripts/test/blaxel-worker-lease.mjs b/scripts/test/blaxel-worker-lease.mjs index 78664072..b073929a 100644 --- a/scripts/test/blaxel-worker-lease.mjs +++ b/scripts/test/blaxel-worker-lease.mjs @@ -15,7 +15,7 @@ export function shardScratchPath({ tmpRoot = "/tmp", lockId, shardIndex, attempt export function workerLeaseCommand(input) { const tmpRoot = input.tmpRoot ?? "/tmp"; const source = `${cleanWorkerTemporaryDirectories.toString()}\n${workerLeaseOperation.toString()}\nworkerLeaseOperation(JSON.parse(Buffer.from(process.argv[1], 'base64url'))).then(result => { console.log(JSON.stringify(result)); if (!result.ok) process.exitCode = 42; }).catch(error => { console.error(error.message); process.exitCode = 2; });`; - return `flock -w 20 -o ${quote(`${tmpRoot}/tv-testshard.guard`)} node -e ${quote(source)} ${quote(Buffer.from(JSON.stringify({ ...input, scratch: input.scratch ?? shardScratchPath(input) })).toString("base64url"))}`; + return `flock -w 20 -F ${quote(`${tmpRoot}/tv-testshard.guard`)} node -e ${quote(source)} ${quote(Buffer.from(JSON.stringify({ ...input, scratch: input.scratch ?? shardScratchPath(input) })).toString("base64url"))}`; } export function workerEntryCommand(input, script) { @@ -118,12 +118,13 @@ export async function workerLeaseOperation(input, hooks = {}) { if (record?.deleting) return operation === "delete" ? { ok: true, deleting: true } : denied("worker reserved for deletion"); if (exists && (held || input.audit)) return denied("worker lease held"); let recovery = null; + let recoveryCleanup = null; if (operation === "delete" && valid && expired) { // Deleting the sandbox is itself cleanup; do not require successful signals. } else if (valid && expired) { const processes = await stopOwners(record.lockId); if (!await auditClean()) return denied("other live or stale owners remain"); - await clean(null); + recoveryCleanup = await clean(null); recovery = { kind: "recovery", previous: { runId: record.runId, lockId: record.lockId, worker: input.worker ?? "unknown", lastRenewal: record.lastRenewal, recoverAfter: record.recoverAfter }, cleanup: "confirmed", discardedReports: true, processes: processes.slice(0, 128), omittedProcesses: Math.max(0, processes.length - 128) }; } else if (!input.audit && !await auditClean()) return denied("live or stale owners remain"); if (operation === "delete") { @@ -137,7 +138,7 @@ export async function workerLeaseOperation(input, hooks = {}) { const parent = path.dirname(scratch); if (fs.existsSync(parent) && fs.lstatSync(parent).isSymbolicLink()) throw new Error("scratch root is a symlink"); fs.mkdirSync(path.join(scratch, "tmp"), { recursive: true, mode: 0o700 }); - record = { lockId: input.lockId, runId: input.runId, shardIndex: input.shardIndex, attemptId: input.attemptId, owner: input.owner, acquiredAt, lastRenewal: acquiredAt, recoverAfter: acquiredAt + ttlMs, expiresAt: 0, scratch, audit: Boolean(input.audit), dispatch: null, recovery }; + record = { lockId: input.lockId, runId: input.runId, shardIndex: input.shardIndex, attemptId: input.attemptId, owner: input.owner, acquiredAt, lastRenewal: acquiredAt, recoverAfter: acquiredAt + ttlMs, expiresAt: 0, scratch, audit: Boolean(input.audit), dispatch: null, recovery, recoveryCleanup, summaryIdentity: input.summaryIdentity }; write(record); return { ok: true, record }; } @@ -154,7 +155,7 @@ export async function workerLeaseOperation(input, hooks = {}) { fs.rmSync(resultsRoot, { recursive: true, force: true }); fs.mkdirSync(resultsRoot, { recursive: true }); fs.writeFileSync(path.join(resultsRoot, "attempt.json"), JSON.stringify({ runId: record.runId, shardIndex: record.shardIndex, attemptId: record.attemptId, lockId: record.lockId })); - fs.writeFileSync(path.join(resultsRoot, "temporary-cleanup.json"), JSON.stringify(cleanup)); + fs.writeFileSync(path.join(resultsRoot, "temporary-cleanup.json"), JSON.stringify(record.recoveryCleanup ? { ...cleanup, recovery: record.recoveryCleanup } : cleanup)); if (record.recovery) { // First evidence in a freshly reset tree. The bounded receipt is also // accounted for by the surface writer's shared attempt budget. @@ -194,7 +195,7 @@ export async function workerLeaseOperation(input, hooks = {}) { fs.writeFileSync(setupFile, JSON.stringify(rows)); } const summaryFile = path.join(resultsRoot, `shard-${record.shardIndex}.json`); - if (!fs.existsSync(summaryFile)) fs.writeFileSync(summaryFile, JSON.stringify({ status: "infra-failed", failureKind: "worker-execution", failureStep: "interrupted-before-summary", failureMessage: "The owned process ended without a finalized shard summary.", shardIndex: record.shardIndex, tasks: [], neverProduced: ["final shard summary"] })); + if (!fs.existsSync(summaryFile)) fs.writeFileSync(summaryFile, JSON.stringify({ ...record.summaryIdentity, collectedFiles: [], status: "infra-failed", failureKind: "worker-execution", failureStep: "interrupted-before-summary", failureMessage: "The owned process ended without a finalized shard summary.", shardIndex: record.shardIndex, tasks: [], neverProduced: ["final shard summary"] })); } const cleanup = await clean(record.scratch, true); record.teardown = { confirmed: true, processes, cleanup }; write(record); diff --git a/scripts/test/resource-diagnostics.mjs b/scripts/test/resource-diagnostics.mjs index ee138bba..5d1d35c7 100644 --- a/scripts/test/resource-diagnostics.mjs +++ b/scripts/test/resource-diagnostics.mjs @@ -189,7 +189,7 @@ export function writeResourceDiagnostic(root, name, input, { write = atomicWrite if (field) record[field] = values.slice(0, count); else if (record.kernel) record.kernel.lines = values.slice(0, count); if (Buffer.byteLength(serialize(record)) <= budget) low = count; else high = count - 1; } - if (field) { record[field] = values.slice(0, low); record.omittedProcesses = values.length - low; } + if (field) { record[field] = values.slice(0, low); record.omittedProcesses = (integer(input.omittedProcesses) ?? 0) + values.length - low; } else if (record.kernel) record.kernel.lines = values.slice(0, low); contents = serialize(record); // The omission count itself can add a few bytes at the boundary. @@ -208,6 +208,7 @@ function cleanRecord(input) { ...base, previous: select(input.previous, ["runId", "lockId", "worker", "lastRenewal", "recoverAfter"]), cleanup: token(input.cleanup), discardedReports: Boolean(input.discardedReports), + omittedProcesses: integer(input.omittedProcesses) ?? 0, processes: (input.processes ?? []).map((entry) => ({ pid: integer(entry.pid), startTime: token(entry.startTime), command: executable(entry.command), outcome: token(entry.outcome) })), }; return { diff --git a/test/repo/blaxel-worker-lease.test.ts b/test/repo/blaxel-worker-lease.test.ts index fb61934b..ed582ba4 100644 --- a/test/repo/blaxel-worker-lease.test.ts +++ b/test/repo/blaxel-worker-lease.test.ts @@ -67,9 +67,10 @@ test.skipIf(process.platform !== "linux")("competing acquisitions serialize, and const owner = f.record().lockId; rmSync(path.dirname(f.recordPath), { recursive: true }); const ready = path.join(f.input.tmpRoot, "guard-ready"); - const child = spawn("flock", ["-o", path.join(f.input.tmpRoot, "tv-testshard.guard"), process.execPath, "-e", `require('fs').writeFileSync(${JSON.stringify(ready)}, 'ready');setInterval(()=>{},1000)`], { detached: true, stdio: "ignore", env: process.env }); + const child = spawn("flock", ["-F", path.join(f.input.tmpRoot, "tv-testshard.guard"), process.execPath, "-e", `require('fs').writeFileSync(${JSON.stringify(ready)}, String(process.pid));setInterval(()=>{},1000)`], { detached: true, stdio: "ignore", env: process.env }); try { await until(() => existsSync(ready)); + expect(Number(readFileSync(ready, "utf8"))).toBe(child.pid); child.kill("SIGKILL"); await command(owner); expect(f.record().lockId).toBe(owner); diff --git a/test/repo/test-runner-provider.test.ts b/test/repo/test-runner-provider.test.ts index 4921d443..4b9f9a79 100644 --- a/test/repo/test-runner-provider.test.ts +++ b/test/repo/test-runner-provider.test.ts @@ -347,7 +347,7 @@ describe("provider result normalization", () => { expect(source).toContain("await finishLease({"); expect(source).toContain('teardown: () => confirmTeardown(lease)'); expect(source).not.toContain("release_lock()"); - const ownerCleanupSource = source.slice(source.indexOf("async function leaseOperation"), source.indexOf("async function acquireAny")); + const ownerCleanupSource = source.slice(source.indexOf("async function leaseOperation"), source.indexOf("function acquireAny")); expect(ownerCleanupSource.indexOf('. "$NVM_DIR/nvm.sh"')).toBeLessThan(ownerCleanupSource.indexOf('"set -u"')); const nvmFixture = mkdtempSync(path.join(os.tmpdir(), "tv-provider-nvm-nounset-")); @@ -1008,6 +1008,13 @@ describe("provider result normalization", () => { }; expect(validateDownloadedShardSummary(summary, plan, 1)).toBeNull(); + for (const missing of ["test execution", "final shard summary"]) { + const incomplete = { ...summary, status: "infra-failed", failureKind: "setup", tasks: [], collectedFiles: [], neverProduced: [missing] }; + expect(validateDownloadedShardSummary(incomplete, plan, 1)).toBeNull(); + expect(classifyDownloadedShardStatus({ shardSummary: incomplete, exitCode: 1 })).toBe("infra-failed"); + expect(validateDownloadedShardSummary({ ...incomplete, planId: "other" }, plan, 1)).toContain("planId mismatch"); + expect(validateDownloadedShardSummary({ ...incomplete, status: "passed" }, plan, 1)).toContain("task assignments disagree"); + } expect(validateDownloadedShardSummary({ ...summary, status: "passed" }, plan, 1)).toContain("reports passed with a skipped assigned task"); expect(validateDownloadedShardSummary({ ...summary, diff --git a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts index a6069f1c..69ffee14 100644 --- a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts +++ b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts @@ -13,7 +13,8 @@ const planned = ["--tag", "blaxel-harness-fixture", "--shards", "2", "--allow-sh const targeted = ["--surface", "experiment:blaxel-harness-fixtures"]; const resultsRoot = "/workspace/television/.testshards/results"; const modePath = "/tmp/rose-blaxel-harness-mode.json"; -const cli = path.resolve("scripts/test/cli.mjs"); +const repositoryRoot = path.resolve(import.meta.dirname, "../../../.."); +const cli = path.join(repositoryRoot, "scripts/test/cli.mjs"); const children = new Set(); const quote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`; const pause = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); @@ -27,7 +28,7 @@ async function remoteJson(worker: SandboxInstance, file: string) { try { return function command(root: string, name: string, args: string[], env: NodeJS.ProcessEnv = {}) { const log = path.join(root, `${name}.log`); - const child = spawn(process.execPath, [cli, ...args], { cwd: process.cwd(), detached: true, env: {...process.env, ...env}, stdio: ["ignore", "pipe", "pipe"] }); + const child = spawn(process.execPath, [cli, ...args], { cwd: repositoryRoot, detached: true, env: {...process.env, ...env}, stdio: ["ignore", "pipe", "pipe"] }); children.add(child); const chunks: Buffer[] = []; child.stdout!.on("data", data => chunks.push(data)); child.stderr!.on("data", data => chunks.push(data)); @@ -56,7 +57,7 @@ function retained(runDir: string) { } function assertDiagnostics(runDir: string, expectedRecovery = false) { const {provider, normalized} = retained(runDir); - const sidecars = normalized.surfaces.flatMap((surface: any) => surface.resourceDiagnosticsPaths ?? []) as string[]; + const sidecars = [...new Set(normalized.surfaces.flatMap((surface: any) => surface.resourceDiagnosticsPaths ?? []))]; expect(sidecars.length).toBeGreaterThan(0); const records = sidecars.map(file => readJson(path.resolve(runDir, file))); for (const record of records) { @@ -79,8 +80,8 @@ test("qualifies exclusive retrieval, active and quiet recovery, temporary placem if (process.env.TV_BLAXEL_HARNESS_COORDINATOR !== coordinatorName || process.env.TV_TEST_SHARD_OWNER) throw Error("Run only on the dedicated Blaxel coordinator, outside every leased worker"); const coordinator = await SandboxInstance.get(coordinatorName); expect(coordinator.metadata.labels?.purpose).not.toBe("testshards"); - const root = mkdtempSync(path.resolve(".test-runs/blaxel-live-")); - writeFileSync(path.join(root, "identity.json"), JSON.stringify({coordinatorName, pool, revision: readFileSync(".git/HEAD", "utf8").trim()})); + const root = mkdtempSync(path.join(repositoryRoot, ".test-runs/blaxel-live-")); + writeFileSync(path.join(root, "identity.json"), JSON.stringify({coordinatorName, pool, revision: readFileSync(path.join(repositoryRoot, ".git/HEAD"), "utf8").trim()})); let workers: SandboxInstance[] = []; try { // The caller retained earlier development reports before this fresh-pool run. From 7ffa179c76e346b770f65f22d84a2e961b6223fb Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:28:56 +0000 Subject: [PATCH 42/64] Tighten live qualification and release failed architecture reservations --- scripts/run-blaxel-testshards.mjs | 12 +++--- test/repo/blaxel-lease-contracts.test.ts | 3 +- .../blaxel-harness/driver/harness.test.ts | 37 ++++++++++++++++--- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index ebfa1004..7e238759 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -141,6 +141,7 @@ async function runStaleOwnerAudit() { const sandboxes = rotatePool(await listPoolSandboxes(), `${runId}:stale-owner-audit`, 1); if (sandboxes.length === 0) fail(`No deployed x64 Blaxel worker is available in pool ${pool}.`); const lease = await acquireAny(sandboxes, { shardIndex: 1, slot: 0, requestedWorkers: 1, verifyArchitecture: false }); + if (shuttingDown) await shutdownPromise; if (!lease) fail(`No available Blaxel worker in pool ${pool} for the read-only stale-owner audit.`); const { command, args } = buildLinuxStaleOwnerAuditInvocation(); @@ -364,12 +365,13 @@ async function acquireWorker(sandboxes, { shardIndex, slot, requestedWorkers, ve renew: () => leaseOperation(lease, "renew"), onLost: (error) => { lease.lost = error; console.error(`Lease renewal lost on ${sandbox.metadata.name}: ${error.message}`); }, }); - if (verifyArchitecture && !(await verifySandboxArchitecture(sandbox))) { - await stopAndReleaseLease(lease, { killProcess: false }); - activeLeases.delete(lockId); - continue; + try { + if (!verifyArchitecture || await verifySandboxArchitecture(sandbox)) return lease; + } catch (error) { + console.error(`Architecture check failed on ${sandbox.metadata.name}: ${error.message}`); } - return lease; + await stopAndReleaseLease(lease, { killProcess: false }); + activeLeases.delete(lockId); } return null; } diff --git a/test/repo/blaxel-lease-contracts.test.ts b/test/repo/blaxel-lease-contracts.test.ts index 2c3f9128..96952cd3 100644 --- a/test/repo/blaxel-lease-contracts.test.ts +++ b/test/repo/blaxel-lease-contracts.test.ts @@ -1,13 +1,14 @@ import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, existsSync, symlinkSync, lstatSync } from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, expect, test, vi } from "vitest"; +import { afterEach, expect, test as platformTest, vi } from "vitest"; import { workerLeaseOperation, cleanWorkerTemporaryDirectories } from "../../scripts/test/blaxel-worker-lease.mjs"; import { finishLease, renewLease } from "../../scripts/test/blaxel-lease-lifecycle.mjs"; import { publishWorker, deleteWorker } from "../../scripts/test/blaxel-pool-lifecycle.mjs"; // Contract clocks, owner audits/signals and SDK operations are substitutions. // Filesystem assertions cross the real cleaner; the live driver proves transport. +const test = platformTest.skipIf(process.platform !== "linux"); const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); function fixture(record?: unknown) { diff --git a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts index 69ffee14..ba041cd3 100644 --- a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts +++ b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts @@ -60,14 +60,22 @@ function assertDiagnostics(runDir: string, expectedRecovery = false) { const sidecars = [...new Set(normalized.surfaces.flatMap((surface: any) => surface.resourceDiagnosticsPaths ?? []))]; expect(sidecars.length).toBeGreaterThan(0); const records = sidecars.map(file => readJson(path.resolve(runDir, file))); - for (const record of records) { + for (const [index, record] of records.entries()) { + expect(provider.shards.some((shard: any) => shard.shardIndex === record.identity.shardIndex)).toBe(true); expect(record.identity.runId).toBe(provider.runId); expect(record.identity.attemptId).toContain(provider.runId); if (record.kind === "surface") { - expect(record.start).toBeTruthy(); expect(record.end).toBeTruthy(); - expect(["available", "unavailable"]).toContain(record.kernel?.status ?? "unavailable"); + expect(record.identity.surfaceId).toBe("experiment:blaxel-harness-fixtures"); + expect(record.start.status).toBe("observed"); expect(record.end.status).toBe("observed"); + expect(record.nativeEvidence.status).toBe("complete"); + for (const reference of [record.nativeEvidence.resultPath, record.nativeEvidence.attemptResultPath].filter(Boolean)) { + expect(existsSync(path.resolve(runDir, path.dirname(sidecars[index]), reference))).toBe(true); + } + if (record.nativeEvidence.failedAttempt) expect(["available", "unavailable"]).toContain(record.kernel?.status); + if (record.kernel?.status === "unavailable") expect(record.kernel.reason).toBeTruthy(); } } + expect(records.some(record => record.nativeEvidence?.recovered === true)).toBe(true); if (expectedRecovery) { const receipts = records.filter(record => record.kind === "recovery"); expect(receipts.length).toBe(2); @@ -99,6 +107,7 @@ test("qualifies exclusive retrieval, active and quiet recovery, temporary placem const first = run(root, "planned", planned, gate); const waiting = await gates(gate); const before = await Promise.all(workers.map(worker => remoteJson(worker, "/tmp/tv-testshard.lock/owner.json"))); + const reportsBefore = await Promise.all(workers.map((worker, index) => worker.fs.read(`${resultsRoot}/shard-${before[index].shardIndex}.json`))); await until(async () => { const after = await Promise.all(workers.map(worker => remoteJson(worker, "/tmp/tv-testshard.lock/owner.json"))); return after.every((record, index) => record.lastRenewal > before[index].lastRenewal) ? after : null; @@ -108,12 +117,15 @@ test("qualifies exclusive retrieval, active and quiet recovery, temporary placem expect(retained(await competed.directory()).summary.run.infraStatus).toBe("incomplete"); expect(await poolCommand(root, "held-delete", "delete")).toBe(1); for (const worker of workers) expect((await remoteJson(worker, "/tmp/tv-testshard.lock/owner.json")).runId).toBe(before[0].runId); + const reportsAfter = await Promise.all(workers.map((worker, index) => worker.fs.read(`${resultsRoot}/shard-${before[index].shardIndex}.json`))); + expect(reportsAfter).toEqual(reportsBefore); writeFileSync(path.join(gate, "continue"), "continue"); expect(await first.done).toBe(1); // intentional native fixture failure const firstDir = await first.directory(); const firstResult = retained(firstDir); expect(firstResult.summary.run.infraStatus).toBe("completed"); expect(firstResult.summary.counts.testsFailed).toBe(1); expect(firstResult.summary.counts.testsFlakyRecovered).toBe(1); assertDiagnostics(firstDir); + const placements: any[] = []; for (const shard of firstResult.provider.shards) { expect(shard.remoteScript).toBe(`${shard.scratch}/worker.sh`); expect(shard.setupTimings.some((phase: any) => phase.name === "temporary-cleanup")).toBe(true); @@ -123,8 +135,13 @@ test("qualifies exclusive retrieval, active and quiet recovery, temporary placem expect(cleanup.removed["television-web-bundle-"].directories).toBe(1); expect(cleanup.removed["stale-scratch"].directories).toBe(1); const placement = path.join(artifact, "temporary-placement.json"); - if (existsSync(placement)) expect(readJson(placement).descriptors.some((entry: any) => entry.path.endsWith(" (deleted)"))).toBe(true); + if (existsSync(placement)) placements.push(readJson(placement)); } + expect(placements).toHaveLength(1); + expect(placements[0].browserVersion).toMatch(/^\d+\./); + expect(placements[0].executable).toContain("/home/playwright/.cache/ms-playwright/"); + expect(placements[0].nodeDir.startsWith(placements[0].tmp + "/")).toBe(true); + expect(placements[0].descriptors.some((entry: any) => entry.path.endsWith(" (deleted)"))).toBe(true); for (const worker of workers) { const ownGate = waiting.find(entry => entry.sandbox === worker.metadata.name)!; expect(await remoteJson(worker, "/tmp/tv-testshard.lock/owner.json")).toBeNull(); @@ -136,6 +153,7 @@ test("qualifies exclusive retrieval, active and quiet recovery, temporary placem expect(await target.done).toBe(1); expect(retained(await target.directory()).summary.run.infraStatus).toBe("completed"); assertDiagnostics(await target.directory()); + expect(existsSync(path.join(await target.directory(), "provider/blaxel/artifacts/shard-1/temporary-placement.json"))).toBe(true); for (const scenario of ["active", "quiet"]) { for (const worker of workers) await worker.fs.write(modePath, JSON.stringify({mode: scenario === "active" ? "active" : "normal"})); @@ -152,12 +170,21 @@ test("qualifies exclusive retrieval, active and quiet recovery, temporary placem process.kill(coordinatorPid, "SIGKILL"); // actual renewer, not just its shard expect(await abandoned.done).not.toBe(0); for (const worker of workers) await worker.fs.write(modePath, JSON.stringify({mode: "normal"})); - await until(async () => Date.now() > Math.max(...owners.map(record => record.recoverAfter)) + 500, 90_000); + owners = await until(async () => { + const current = await Promise.all(workers.map(worker => remoteJson(worker, "/tmp/tv-testshard.lock/owner.json"))); + return current.every((record, index) => record?.lockId === owners[index].lockId && Date.now() > record.recoverAfter + 500) ? current : null; + }, 120_000); const recovered = run(root, `${scenario}-recovered`, planned); expect(await recovered.done).toBe(1); const directory = await recovered.directory(); expect(retained(directory).summary.run.infraStatus).toBe("completed"); const records = assertDiagnostics(directory, true); + for (const receipt of records.filter(record => record.kind === "recovery")) { + const previous = owners.find(owner => owner.lockId === receipt.previous.lockId); + expect(previous).toBeTruthy(); + expect(receipt.previous.lastRenewal).toBe(previous.lastRenewal); + expect(receipt.previous.recoverAfter).toBe(previous.recoverAfter); + } if (scenario === "active") expect(records.filter(record => record.kind === "recovery").every(record => record.processes.length > 0)).toBe(true); writeFileSync(path.join(root, `${scenario}-recovery-evidence.json`), JSON.stringify({owners, recoveredRun: directory})); } From 49cfe0c31b99a635e48730860c9b078ba291d15f Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:33:29 +0000 Subject: [PATCH 43/64] Revalidate cleanup targets and record private qualification progress --- docs/working/test-harness-resource-cleanup-plan.md | 2 +- .../test-harness-resource-cleanup-slice-3.md | 13 +++++++++++++ scripts/test/blaxel-worker-lease.mjs | 2 ++ test/repo/blaxel-lease-contracts.test.ts | 13 +++++++++++-- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/docs/working/test-harness-resource-cleanup-plan.md b/docs/working/test-harness-resource-cleanup-plan.md index 48b88a6d..f667a68a 100644 --- a/docs/working/test-harness-resource-cleanup-plan.md +++ b/docs/working/test-harness-resource-cleanup-plan.md @@ -1,6 +1,6 @@ # Test-harness resource cleanup: implementation plan -Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; [slice 1 has converged](test-harness-resource-cleanup-slice-1-review-r1.md), and [slice 2 is implemented](test-harness-resource-cleanup-slice-2.md) for independent review. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. +Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; [slice 1 has converged](test-harness-resource-cleanup-slice-1-review-r1.md), and [slice 2 has converged](test-harness-resource-cleanup-slice-2-review-r1.md). [Slice 3](test-harness-resource-cleanup-slice-3.md) is in implementation and private qualification. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. ## Slice judgment diff --git a/docs/working/test-harness-resource-cleanup-slice-3.md b/docs/working/test-harness-resource-cleanup-slice-3.md index a2b75b23..140b029a 100644 --- a/docs/working/test-harness-resource-cleanup-slice-3.md +++ b/docs/working/test-harness-resource-cleanup-slice-3.md @@ -71,3 +71,16 @@ and by confirmed teardown after a killed shell. The correction adds that fixture boundary without changing repository selection or checkout behavior. The new leases remain held until their ordinary deadlines (about 19:31 UTC); no lock was stolen or deadline edited. + + +Read-only inventory at 19:29:39 UTC counted 146 sandboxes, all reported DEPLOYED, +including the two `rose-cleanup` workers and `tv-cleanup-coordinator-rose` outside +the pool. This does not establish active-work concurrency or an account-specific +quota; it does establish that the historical 75 figure is not a sandbox-count +limit here. Fresh live qualification replaces two owned workers after deletion, +without increasing the private worker count or touching shared workers. + +`c7482876` passed lint and type checking on the dedicated coordinator. Its next +lease run deliberately supplies `--lock-ttl-seconds 60` to qualify recovery from +the retained failed attempt; native and infrastructure retry budgets stay at their +defaults. The following contract run uses the ordinary recovery default. diff --git a/scripts/test/blaxel-worker-lease.mjs b/scripts/test/blaxel-worker-lease.mjs index b073929a..b591cffe 100644 --- a/scripts/test/blaxel-worker-lease.mjs +++ b/scripts/test/blaxel-worker-lease.mjs @@ -261,6 +261,8 @@ export async function cleanWorkerTemporaryDirectories({ tmpRoot = "/tmp", curren const stat = lstat(file); if (!stat.isDirectory() || stat.isSymbolicLink()) { report.skips.push({ prefix, reason: "not a real directory" }); continue; } const bytes = allocated(file, lstat(tmpRoot).dev); + const currentIdentity = lstat(file); + if (!currentIdentity.isDirectory() || currentIdentity.isSymbolicLink() || currentIdentity.dev !== stat.dev || currentIdentity.ino !== stat.ino) throw new Error("directory identity changed before removal"); remove(file); const counts = report.removed[prefix] ??= { directories: 0, allocatedBytes: 0 }; counts.directories++; counts.allocatedBytes += bytes; diff --git a/test/repo/blaxel-lease-contracts.test.ts b/test/repo/blaxel-lease-contracts.test.ts index 96952cd3..29798170 100644 --- a/test/repo/blaxel-lease-contracts.test.ts +++ b/test/repo/blaxel-lease-contracts.test.ts @@ -77,9 +77,18 @@ test("only known real directories are removed, with allocated-byte evidence and }); test("mount boundaries and incomplete removal remain reported failures without reclaimed counts", async () => { - for (const failure of ["mount", "remove"]) { + for (const failure of ["mount", "identity", "remove"]) { const f = fixture(); const dir = path.join(f.tmpRoot, "television-web-bundle-old"); mkdirSync(dir); - const probes = failure === "mount" ? { lstat: (file: string) => { const stat = lstatSync(file); if (file === dir) Object.defineProperty(stat, "dev", { value: stat.dev + 1 }); return stat; } } : { remove: () => { throw Object.assign(Error("incomplete"), { code: "EIO" }); } }; + let directoryReads = 0; + const probes = failure !== "remove" ? { lstat: (file: string) => { + const stat = lstatSync(file); + if (file === dir) { + directoryReads++; + if (failure === "mount") Object.defineProperty(stat, "dev", { value: stat.dev + 1 }); + if (failure === "identity" && directoryReads === 3) Object.defineProperty(stat, "ino", { value: stat.ino + 1 }); + } + return stat; + } } : { remove: () => { throw Object.assign(Error("incomplete"), { code: "EIO" }); } }; const result = await cleanWorkerTemporaryDirectories({ tmpRoot: f.tmpRoot, current: null }, probes); expect(result.ok).toBe(false); expect(result.removed).toEqual({}); expect(result.errors).toHaveLength(1); expect(existsSync(dir)).toBe(true); } From 720cef01c75e26f3ab29312c55b4502c2067cded Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:37:43 +0000 Subject: [PATCH 44/64] Update coordinator wiring assertions for retained shutdown evidence --- test/repo/test-runner-provider.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/repo/test-runner-provider.test.ts b/test/repo/test-runner-provider.test.ts index 4b9f9a79..7cf9766e 100644 --- a/test/repo/test-runner-provider.test.ts +++ b/test/repo/test-runner-provider.test.ts @@ -121,9 +121,9 @@ describe("provider result normalization", () => { }); const source = readFileSync(path.join(process.cwd(), "scripts/run-blaxel-testshards.mjs"), "utf8"); - expect(source).toContain("const dispatch = await transferShardInputs({"); + expect(source).toContain("const dispatch = await (lease.pendingWrites = transferShardInputs({"); expect(source).toContain("return dispatchFailureResult({ lease, outputDir, startedAt, logPath, dispatch });"); - expect(source.indexOf("const dispatch = await transferShardInputs({")).toBeLessThan(source.indexOf("sandbox.process.exec({", source.indexOf("async function runShard"))); + expect(source.indexOf("const dispatch = await (lease.pendingWrites = transferShardInputs({")).toBeLessThan(source.indexOf("sandbox.process.exec({", source.indexOf("async function runShard"))); }); test("preserves completed, running, and pending shard evidence before signal cleanup", async () => { @@ -237,7 +237,7 @@ describe("provider result normalization", () => { const source = readFileSync(path.join(process.cwd(), "scripts/run-blaxel-testshards.mjs"), "utf8"); expect(source).toContain("interruptedSnapshot = buildInterruptedReportSnapshot(signal)"); expect(source).toContain("releaseLeases: async () => {"); - expect(source).toContain("finalizeReport: () => writeInterruptedReportSnapshot(interruptedSnapshot)"); + expect(source).toContain("finalizeReport: () => writeInterruptedReportSnapshot(buildInterruptedReportSnapshot(signal) ?? interruptedSnapshot)"); expect(source).toContain("completedResults: activeAttempt.completedResults"); expect(source).toContain("partialResults: activeAttempt.partialResults"); expect(source).toContain("executionRequested: activeAttempt.executionRequested"); From 5b23a426409931b4472d567ad18a57258de03199 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:42:36 +0000 Subject: [PATCH 45/64] Cover audit residue preservation and unknown dispatch retention --- test/repo/blaxel-lease-contracts.test.ts | 9 +++++++++ test/repo/blaxel-worker-lease.test.ts | 2 ++ 2 files changed, 11 insertions(+) diff --git a/test/repo/blaxel-lease-contracts.test.ts b/test/repo/blaxel-lease-contracts.test.ts index 29798170..4bf38244 100644 --- a/test/repo/blaxel-lease-contracts.test.ts +++ b/test/repo/blaxel-lease-contracts.test.ts @@ -48,6 +48,15 @@ test("legacy, incomplete, unreadable and audit leases fail closed without clean } const f = fixture({ lockId: "legacy", expiresAt: 100 }); expect(await workerLeaseOperation(f.input, { now: () => 3000, scanOwners: async () => [] })).toMatchObject({ ok: false }); + const audit = fixture(); + const residue = path.join(audit.tmpRoot, "television-web-bundle-preserve"); mkdirSync(residue); + const staleScratch = path.join(audit.tmpRoot, "tv-testshards/stale"); mkdirSync(staleScratch, { recursive: true }); + const auditInput = { ...audit.input, audit: true }; + const auditHooks = { scanOwners: async () => { throw Error("audit must observe existing owners before cleanup"); }, stopOwners: async () => [] }; + expect((await workerLeaseOperation(auditInput, auditHooks)).ok).toBe(true); + expect((await workerLeaseOperation({ ...auditInput, operation: "prepare" }, auditHooks)).ok).toBe(true); + expect((await workerLeaseOperation({ ...auditInput, operation: "teardown" }, auditHooks)).ok).toBe(true); + expect(existsSync(residue)).toBe(true); expect(existsSync(staleScratch)).toBe(true); }); test("cleanup failures keep the old identity, while explicit deletion reserves an expired worker", async () => { diff --git a/test/repo/blaxel-worker-lease.test.ts b/test/repo/blaxel-worker-lease.test.ts index ed582ba4..27a04e34 100644 --- a/test/repo/blaxel-worker-lease.test.ts +++ b/test/repo/blaxel-worker-lease.test.ts @@ -85,6 +85,8 @@ test.skipIf(process.platform !== "linux")("an obsolete delayed entry cannot touc writeFileSync(script, `touch ${shell(sentinel)}\n`); expect(f.run({ operation: "dispatch", processName }).status).toBe(0); const entry = workerEntryCommand({ ...f.input, processName }, script); + expect(f.run({ operation: "teardown" }).status).toBe(42); + expect(f.run({ operation: "release", retrieved: true }).status).toBe(42); // The real entry command is delayed until after the old identity is replaced. writeFileSync(f.recordPath, JSON.stringify({ ...f.record(), recoverAfter: 1 })); expect(f.run({ lockId: "new-lock", attemptId: "new-attempt" }).status).toBe(0); From 9d197e8f3eb691c37830f9bf518986b087127bb8 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:47:36 +0000 Subject: [PATCH 46/64] Wait for accepted private worker deletion to become observable --- .../blaxel-harness/driver/harness.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts index ba041cd3..9a31b925 100644 --- a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts +++ b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts @@ -44,7 +44,14 @@ function run(root: string, name: string, selection: string[], gate?: string) { return {...task, directory: async () => until(async () => existsSync(pointer) ? readFileSync(pointer, "utf8").trim() : null)}; } async function poolCommand(root: string, name: string, operation: string) { - return command(root, name, ["pool", "blaxel", operation, "--pool", pool, ...(operation === "ensure" ? ["--size", "2"] : [])]).done; + const code = await command(root, name, ["pool", "blaxel", operation, "--pool", pool, ...(operation === "ensure" ? ["--size", "2"] : [])]).done; + // The API accepts deletion before its listing stops returning the worker. + // Observe absence before fresh provisioning or claiming the pool is gone. + if (operation === "delete" && code === 0) await until(async () => { + if ((await listBlaxelPoolSandboxes({pool, arch: "x64"})).length === 0) return true; + await pause(1000); return null; + }); + return code; } async function gates(gate: string) { return until(async () => [1,2].every(index => existsSync(path.join(gate, `shard-${index}.json`))) ? [1,2].map(index => readJson(path.join(gate, `shard-${index}.json`))) : null); From d32457f79d063230acf5f149f4aa4b890ea4bd84 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:04:55 +0000 Subject: [PATCH 47/64] Follow retry artifact paths and limit short leases to crash experiments --- .../blaxel-harness/driver/harness.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts index 9a31b925..759ae5f9 100644 --- a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts +++ b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts @@ -38,9 +38,9 @@ function command(root: string, name: string, args: string[], env: NodeJS.Process }); return {child, done, log}; } -function run(root: string, name: string, selection: string[], gate?: string) { +function run(root: string, name: string, selection: string[], gate?: string, recoverySeconds?: number) { const pointer = path.join(root, `${name}.run`); - const task = command(root, name, ["blaxel", "--pool", pool, ...selection, "--lock-ttl-seconds", "60", "--run-dir-output", pointer], gate ? {TV_TEST_RUNNER_SELFTEST: "1", TV_BLAXEL_DOWNLOAD_GATE: gate} : {}); + const task = command(root, name, ["blaxel", "--pool", pool, ...selection, ...(recoverySeconds ? ["--lock-ttl-seconds", String(recoverySeconds)] : []), "--run-dir-output", pointer], gate ? {TV_TEST_RUNNER_SELFTEST: "1", TV_BLAXEL_DOWNLOAD_GATE: gate} : {}); return {...task, directory: async () => until(async () => existsSync(pointer) ? readFileSync(pointer, "utf8").trim() : null)}; } async function poolCommand(root: string, name: string, operation: string) { @@ -136,7 +136,7 @@ test("qualifies exclusive retrieval, active and quiet recovery, temporary placem for (const shard of firstResult.provider.shards) { expect(shard.remoteScript).toBe(`${shard.scratch}/worker.sh`); expect(shard.setupTimings.some((phase: any) => phase.name === "temporary-cleanup")).toBe(true); - const artifact = path.join(firstDir, "provider/blaxel/artifacts", `shard-${shard.shardIndex}`); + const artifact = path.join(shard.outputDir, "artifacts", `shard-${shard.shardIndex}`); const cleanup = readJson(path.join(artifact, "temporary-cleanup.json")); expect(cleanup.removed["television-source-cli-e2e-"].directories).toBe(1); expect(cleanup.removed["television-web-bundle-"].directories).toBe(1); @@ -160,12 +160,13 @@ test("qualifies exclusive retrieval, active and quiet recovery, temporary placem expect(await target.done).toBe(1); expect(retained(await target.directory()).summary.run.infraStatus).toBe("completed"); assertDiagnostics(await target.directory()); - expect(existsSync(path.join(await target.directory(), "provider/blaxel/artifacts/shard-1/temporary-placement.json"))).toBe(true); + const targetShard = retained(await target.directory()).provider.shards[0]; + expect(existsSync(path.join(targetShard.outputDir, "artifacts/shard-1/temporary-placement.json"))).toBe(true); for (const scenario of ["active", "quiet"]) { for (const worker of workers) await worker.fs.write(modePath, JSON.stringify({mode: scenario === "active" ? "active" : "normal"})); const crashGate = path.join(root, `${scenario}-download`); mkdirSync(crashGate); - const abandoned = run(root, `${scenario}-abandoned`, planned, crashGate); + const abandoned = run(root, `${scenario}-abandoned`, planned, crashGate, 60); let owners: any[]; if (scenario === "quiet") { await gates(crashGate); owners = await Promise.all(workers.map(worker => remoteJson(worker, "/tmp/tv-testshard.lock/owner.json"))); } else owners = await until(async () => { From 06b07c0ff0aa683a7fadb4dc8f5fb72decd78d52 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:15:13 +0000 Subject: [PATCH 48/64] Expose evidence retention and require prepared undispatched reports before release --- scripts/run-blaxel-testshards.mjs | 32 +-------------------- scripts/test/blaxel-lease-lifecycle.d.mts | 1 + scripts/test/blaxel-lease-lifecycle.mjs | 35 +++++++++++++++++++++++ test/repo/blaxel-lease-contracts.test.ts | 27 ++++++++++++++++- 4 files changed, 63 insertions(+), 32 deletions(-) diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index 7e238759..8ef1c4e2 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -15,7 +15,7 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import { randomUUID } from "node:crypto"; -import { renewLease, finishLease } from "./test/blaxel-lease-lifecycle.mjs"; +import { renewLease, finishLease, retainAttemptEvidence } from "./test/blaxel-lease-lifecycle.mjs"; import { workerLeaseCommand, workerEntryCommand, shardScratchPath } from "./test/blaxel-worker-lease.mjs"; import path from "node:path"; import process from "node:process"; @@ -1069,36 +1069,6 @@ function confirmTeardown(lease) { return lease.teardownPromise; } -async function retainAttemptEvidence(lease, directory) { - if (!lease.executionRequested) { lease.retrieved = true; return; } - const root = "/workspace/television/.testshards/results"; - const identity = JSON.parse(await lease.sandbox.fs.read(`${root}/attempt.json`)); - for (const key of ["lockId", "runId", "shardIndex", "attemptId"]) if (identity[key] !== lease.input[key]) throw new Error(`attempt evidence identity mismatch: ${key}`); - let evidence; - try { evidence = JSON.parse(await lease.sandbox.fs.read(`${root}/evidence.json`)); } - catch (error) { - if (lease.teardownConfirmed || error?.status !== 404) throw error; - // Teardown uncertainty must not prevent retaining already-produced evidence. - const files = []; - const collect = async (directory) => { - const listing = await lease.sandbox.fs.ls(directory); - for (const file of listing.files ?? []) files.push(path.posix.relative(root, `${directory}/${file.name}`)); - for (const child of listing.subdirectories ?? []) await collect(`${directory}/${child.name}`); - }; - await collect(root); - evidence = { runId: lease.input.runId, attemptId: lease.input.attemptId, files }; - } - if (evidence.runId !== lease.input.runId || evidence.attemptId !== lease.input.attemptId || !Array.isArray(evidence.files)) throw new Error("invalid retained-evidence inventory"); - for (const name of evidence.files) { - // Optional diagnostic reads use their bounded, warning-only transport below. - // They must not make a native result fail or bypass its retention budget. - if (typeof name === "string" && name.startsWith("resources/")) continue; - if (typeof name !== "string" || name.startsWith("/") || name.split("/").includes("..")) throw new Error("invalid evidence path"); - const raw = await lease.sandbox.fs.read(`${root}/${name}`); - const target = path.join(directory, "artifacts", `shard-${lease.shardIndex}`, name); - fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, raw); - } -} async function beforeDownload(lease) { if (process.env.TV_TEST_RUNNER_SELFTEST !== "1" || !process.env.TV_BLAXEL_DOWNLOAD_GATE) return; diff --git a/scripts/test/blaxel-lease-lifecycle.d.mts b/scripts/test/blaxel-lease-lifecycle.d.mts index ab8e5dd3..e0c288a5 100644 --- a/scripts/test/blaxel-lease-lifecycle.d.mts +++ b/scripts/test/blaxel-lease-lifecycle.d.mts @@ -1,2 +1,3 @@ export function renewLease(options: {renew: () => Promise; intervalMs: number; onLost: (error: Error) => void}): () => Promise; export function finishLease(options: {settleWrites: () => Promise; teardown: () => Promise; download: () => Promise; release: () => Promise; assertOwned: () => void; attempts?: number}): Promise; +export function retainAttemptEvidence(lease: {preparationRequested?: boolean; executionRequested?: boolean; retrieved?: boolean; teardownConfirmed?: boolean; shardIndex: number; input: Record; sandbox: {fs: {read(file: string): Promise; ls(directory: string): Promise<{files?: Array<{name: string}>; subdirectories?: Array<{name: string}>}>}}}, directory: string): Promise; diff --git a/scripts/test/blaxel-lease-lifecycle.mjs b/scripts/test/blaxel-lease-lifecycle.mjs index 37abdf0d..29220f6e 100644 --- a/scripts/test/blaxel-lease-lifecycle.mjs +++ b/scripts/test/blaxel-lease-lifecycle.mjs @@ -1,3 +1,6 @@ +import fs from "node:fs"; +import path from "node:path"; + /** Coordinator-local lifecycle; transport operations are supplied at one seam. */ export function renewLease({ renew, intervalMs, onLost }) { let stopped = false; @@ -27,3 +30,35 @@ export async function finishLease({ settleWrites, teardown, download, release, a assertOwned(); await release(); } + +export async function retainAttemptEvidence(lease, directory) { + if (!lease.executionRequested) { lease.retrieved = true; return; } + const root = "/workspace/television/.testshards/results"; + const identity = JSON.parse(await lease.sandbox.fs.read(`${root}/attempt.json`)); + for (const key of ["lockId", "runId", "shardIndex", "attemptId"]) if (identity[key] !== lease.input[key]) throw new Error(`attempt evidence identity mismatch: ${key}`); + let evidence; + try { evidence = JSON.parse(await lease.sandbox.fs.read(`${root}/evidence.json`)); } + catch (error) { + if (lease.teardownConfirmed || error?.status !== 404) throw error; + // Teardown uncertainty must not prevent retaining already-produced evidence. + const files = []; + const collect = async (directory) => { + const listing = await lease.sandbox.fs.ls(directory); + for (const file of listing.files ?? []) files.push(path.posix.relative(root, `${directory}/${file.name}`)); + for (const child of listing.subdirectories ?? []) await collect(`${directory}/${child.name}`); + }; + await collect(root); + evidence = { runId: lease.input.runId, attemptId: lease.input.attemptId, files }; + } + if (evidence.runId !== lease.input.runId || evidence.attemptId !== lease.input.attemptId || !Array.isArray(evidence.files)) throw new Error("invalid retained-evidence inventory"); + for (const name of evidence.files) { + // Optional diagnostic reads use their bounded, warning-only transport below. + // They must not make a native result fail or bypass its retention budget. + if (typeof name === "string" && name.startsWith("resources/")) continue; + if (typeof name !== "string" || name.startsWith("/") || name.split("/").includes("..")) throw new Error("invalid evidence path"); + const raw = await lease.sandbox.fs.read(`${root}/${name}`); + const target = path.join(directory, "artifacts", `shard-${lease.shardIndex}`, name); + fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, raw); + } +} + diff --git a/test/repo/blaxel-lease-contracts.test.ts b/test/repo/blaxel-lease-contracts.test.ts index 4bf38244..e305eb82 100644 --- a/test/repo/blaxel-lease-contracts.test.ts +++ b/test/repo/blaxel-lease-contracts.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, expect, test as platformTest, vi } from "vitest"; import { workerLeaseOperation, cleanWorkerTemporaryDirectories } from "../../scripts/test/blaxel-worker-lease.mjs"; -import { finishLease, renewLease } from "../../scripts/test/blaxel-lease-lifecycle.mjs"; +import { finishLease, renewLease, retainAttemptEvidence } from "../../scripts/test/blaxel-lease-lifecycle.mjs"; import { publishWorker, deleteWorker } from "../../scripts/test/blaxel-pool-lifecycle.mjs"; // Contract clocks, owner audits/signals and SDK operations are substitutions. @@ -165,3 +165,28 @@ test("failed deletion of never-published workers remains retryable without their await deleteWorker(sandbox, ops); expect(ops.reserve).not.toHaveBeenCalled(); expect(ops.update).not.toHaveBeenCalled(); expect(ops.remove).toHaveBeenCalledTimes(2); }); + +test("preparation evidence is downloaded before an undispatched attempt releases", async () => { + const f = fixture(); + const hooks = { scanOwners: async () => [], stopOwners: async () => [] }; + await workerLeaseOperation(f.input, hooks); + await workerLeaseOperation({ ...f.input, operation: "prepare" }, hooks); + writeFileSync(path.join(f.input.resultsRoot, "preparation.txt"), "produced before dispatch"); + const directory = path.join(f.tmpRoot, "download"); + const lease = { preparationRequested: true, executionRequested: false, teardownConfirmed: true, + shardIndex: 1, input: f.input, sandbox: { fs: { + read: async (file: string) => readFileSync(file.replace("/workspace/television/.testshards/results", f.input.resultsRoot), "utf8"), + ls: async () => { throw Error("confirmed teardown must supply its inventory"); }, + } }, + }; + await finishLease({ + settleWrites: async () => {}, assertOwned: () => {}, + teardown: async () => { expect((await workerLeaseOperation({ ...f.input, operation: "teardown" }, hooks)).ok).toBe(true); }, + download: () => retainAttemptEvidence(lease, directory), + release: async () => { + expect(existsSync(path.join(directory, "artifacts/shard-1/preparation.txt"))).toBe(true); + expect(readFileSync(path.join(directory, "artifacts/shard-1/preparation.txt"), "utf8")).toBe("produced before dispatch"); + expect((await workerLeaseOperation({ ...f.input, operation: "release", retrieved: true }, hooks)).ok).toBe(true); + }, + }); +}); From 45202ee9ca9be83d2c52a171d5bfa63369cd7c4c Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:28:12 +0000 Subject: [PATCH 49/64] Retain prepared evidence before release and allow the cold-worker rendezvous budget --- scripts/run-blaxel-testshards.mjs | 14 +++++++++++--- scripts/test/blaxel-lease-lifecycle.mjs | 2 +- .../blaxel-harness/driver/harness.test.ts | 3 ++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index 8ef1c4e2..d51aaea0 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -327,6 +327,7 @@ function hashString(value) { async function leaseOperation(lease, operation, extra = {}) { if (lease.lost) throw lease.lost; + if (operation === "prepare") lease.preparationRequested = true; const body = ["set -eo pipefail", 'export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"', '. "$NVM_DIR/nvm.sh"', "nvm use --silent default", "set -u", workerLeaseCommand({ ...lease.input, operation, ...extra })].join("; "); const result = await lease.sandbox.process.exec({ name: `tv-${operation}-${randomUUID()}`, command: `bash -lc ${shellQuote(body)}`, waitForCompletion: true, timeout: 30 }); const lines = chunkToText(result.logs ?? "").trim().split("\n"); @@ -487,10 +488,17 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE teardown: () => confirmTeardown(lease), download: async () => { await beforeDownload(lease); - const logs = await sandbox.process.logs(processName, "all"); - appendLog(logPath, `\n[coordinator] final worker log:\n${chunkToText(logs)}`); + let logError = null; + if (lease.executionRequested) { + try { + const logs = await sandbox.process.logs(processName, "all"); + appendLog(logPath, `\n[coordinator] final worker log:\n${chunkToText(logs)}`); + } catch (error) { logError = error; } + } failureStep = "report-download"; result = await downloadResult(); + // Retain available artifacts even when a required log read failed. + if (logError) throw logError; lease.retrieved = true; }, release: async () => { await leaseOperation(lease, "release", { retrieved: true }); lease.released = true; }, @@ -1092,7 +1100,7 @@ async function stopAndReleaseLease(lease, { killProcess }) { await confirmTeardown(lease); if (killProcess && lease.workPromise) await lease.workPromise; if (lease.released) return; - if (!lease.executionRequested) lease.retrieved = true; + if (!lease.preparationRequested && !lease.executionRequested) lease.retrieved = true; if (!lease.retrieved) throw new Error("results not fully retained; keeping lease for recovery"); await leaseOperation(lease, "release", { retrieved: true }); lease.released = true; diff --git a/scripts/test/blaxel-lease-lifecycle.mjs b/scripts/test/blaxel-lease-lifecycle.mjs index 29220f6e..46118b60 100644 --- a/scripts/test/blaxel-lease-lifecycle.mjs +++ b/scripts/test/blaxel-lease-lifecycle.mjs @@ -32,7 +32,7 @@ export async function finishLease({ settleWrites, teardown, download, release, a } export async function retainAttemptEvidence(lease, directory) { - if (!lease.executionRequested) { lease.retrieved = true; return; } + if (!lease.preparationRequested && !lease.executionRequested) { lease.retrieved = true; return; } const root = "/workspace/television/.testshards/results"; const identity = JSON.parse(await lease.sandbox.fs.read(`${root}/attempt.json`)); for (const key of ["lockId", "runId", "shardIndex", "attemptId"]) if (identity[key] !== lease.input[key]) throw new Error(`attempt evidence identity mismatch: ${key}`); diff --git a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts index 759ae5f9..bf2d9e8f 100644 --- a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts +++ b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts @@ -54,7 +54,8 @@ async function poolCommand(root: string, name: string, operation: string) { return code; } async function gates(gate: string) { - return until(async () => [1,2].every(index => existsSync(path.join(gate, `shard-${index}.json`))) ? [1,2].map(index => readJson(path.join(gate, `shard-${index}.json`))) : null); + // Cold worker execution is allowed 480 seconds, before teardown and retrieval. + return until(async () => [1,2].every(index => existsSync(path.join(gate, `shard-${index}.json`))) ? [1,2].map(index => readJson(path.join(gate, `shard-${index}.json`))) : null, 600_000); } function retained(runDir: string) { const summary = readJson(path.join(runDir, "summary.json")); From 2dc0e7d606cd9449edfa033d8be54ce9b513fbef Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:39:06 +0000 Subject: [PATCH 50/64] Record slice 3 private qualification and completed proof coverage --- .../test-harness-resource-cleanup-plan.md | 4 +- ...arness-resource-cleanup-review-response.md | 10 + .../test-harness-resource-cleanup-slice-3.md | 241 +++++++++++++++++- proofs/arch/test-runner/blaxel-testshards.md | 30 ++- proofs/arch/test-runner/reporting.md | 4 +- proofs/arch/test-runner/test-registry.md | 2 +- proofs/arch/test-runner/test-runner.md | 2 +- 7 files changed, 265 insertions(+), 28 deletions(-) diff --git a/docs/working/test-harness-resource-cleanup-plan.md b/docs/working/test-harness-resource-cleanup-plan.md index f667a68a..d350f981 100644 --- a/docs/working/test-harness-resource-cleanup-plan.md +++ b/docs/working/test-harness-resource-cleanup-plan.md @@ -1,6 +1,6 @@ # Test-harness resource cleanup: implementation plan -Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; [slice 1 has converged](test-harness-resource-cleanup-slice-1-review-r1.md), and [slice 2 has converged](test-harness-resource-cleanup-slice-2-review-r1.md). [Slice 3](test-harness-resource-cleanup-slice-3.md) is in implementation and private qualification. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. +Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; [slice 1 has converged](test-harness-resource-cleanup-slice-1-review-r1.md), and [slice 2 has converged](test-harness-resource-cleanup-slice-2-review-r1.md). [Slice 3](test-harness-resource-cleanup-slice-3.md) is implemented and privately qualified, awaiting independent review. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. ## Slice judgment @@ -93,4 +93,4 @@ Retrieve all evidence before deleting the dedicated coordinator, and report clea ## Current limits -Slice 1's helper contracts, real framework teardown and five built-application walks have converged. Slice 2's bounded collector, parent lifecycle and retained-link contracts have red/green evidence, with affected-file checks and remaining live-provider obligations recorded in its evidence document. The named private coordinator and two workers exist; credentials, Node and x64/Linux execution are established there. Provider inventory differs from the historical pool count, and account-specific quota and idle-sandbox accounting remain unknown. The coordinator's experiment-driver preflight, renewal/recovery and pool-label behavior, interrupted dispatch, browser temporary placement, diagnostic retention across reuse and recovery, and full-gate duration remain for slice 3. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. +Slice 1's helper contracts, real framework teardown and five built-application walks have converged. Slice 2's diagnostics have converged. Slice 3's focused contracts, Linux seams and live private qualification have passed; its evidence record carries the failed attempts, fixes, final revisions and retained run paths. Both private workers were deleted through the manager and confirmed absent; the dedicated coordinator remains for integrated verification. Slice 3 and integrated review, target reconciliation and the full gate are still outstanding. Provider inventory differs from the historical pool count; account-specific quota, idle-sandbox accounting, the cause of one deliberately shortened lease's timing lapse, and full-gate duration remain unknown. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index f63a4c2c..4dc7dd82 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -275,3 +275,13 @@ A further Primitive warning named `dec_8e3d38a8`, check `td7crdek1b1n4dhajyww5p174s8h23bq`. Its decision text was not read. Per the supervisor's direction, record the unread warning and continue with the converged ordinary Linux fixture seam and private live-driver proof; do not debug the CLI. + +## Slice 3: implemented for independent review + +The final executable checkpoint is `45202ee9`. The [slice 3 record](test-harness-resource-cleanup-slice-3.md) preserves the original renewable-lock red, intermediate serialization/overlay/checkout-reset failures, focused green runs, live-driver corrections, and the late prepared-but-undispatched evidence regression and fix. The complete slice 2 review remains preserved and all three bounded refinements are addressed. + +Ownership now serializes through a short worker guard, renews through downloads, registers dispatch before checkout, audits and stops attributed descendants, and releases only after retained evidence. Expired renewal performs the same cleanup for active and quiet abandoned runs and keeps a bounded receipt. Known temporary prefixes and attempt scratch are reclaimed under that ownership; pool publication and deletion obey the same exclusion. Older timestamp-only records retain the reviewed transition behavior. The preparation-retention fix distinguishes requesting preparation from requesting execution and retains available artifacts even when a required log read fails. + +Seven real Linux lease seam cases and ten lifecycle/cleanup/pool contracts have passed, alongside affected provider, discovery, stale-owner, registry, diagnostic, shard-worker and process-helper files. Lint and type checking passed at the final executable checkpoint. The final full live driver passed on its first native attempt in 4m 31s, with zero outer failures, recovered retries, process leaks or infrastructure failures. Its intentional failed/recovered native fixtures, competing and absent-pool refusals, killed-coordinator reports, receipt links, and final cleanup are retained. The proof citations now name the implemented coverage, resolving the temporary live-provider obligations from slices 1 and 2. + +Both private workers were deleted and confirmed absent; the dedicated coordinator remains for the integrated gate. No tests or static checks ran on the shared host, and no shared-pool experiments occurred. Independent slice review and then integrated review/full verification remain; the draft PR is not ready to merge. No new lasting team decision was made. Primitive's unread conflict ID and incomplete hook scans are recorded in the validation record, and its CLI received no further debugging. diff --git a/docs/working/test-harness-resource-cleanup-slice-3.md b/docs/working/test-harness-resource-cleanup-slice-3.md index 140b029a..1489cb1c 100644 --- a/docs/working/test-harness-resource-cleanup-slice-3.md +++ b/docs/working/test-harness-resource-cleanup-slice-3.md @@ -1,6 +1,8 @@ # Slice 3: exclusive Blaxel worker lifecycle -Status: implementation and private qualification in progress; not ready for review. +Status: implemented and privately qualified; ready for independent slice review. +Final executable checkpoint: `45202ee9`. The final live qualification passed on its +first native attempt in 4m 31s; complete evidence and limits are below. The converged plan and proofs govern the scope. Slice 2 passed at `296391c0`. No tests, builds, lint, type checks, browsers or servers run on the shared host. All executions below use pushed revisions on the private Blaxel coordinator and @@ -20,9 +22,11 @@ fails, so a partial provisioning failure does not require an installed runtime t retry deletion. The live driver and its two native fixture files are explicit experiment surfaces. -They use only the named private pool, shorten recovery to 60 seconds deliberately, -and retain default native and infrastructure retries. Normal runs retain the -1200-second recovery interval. Qualification remains pending below. +They use only the named private pool and retain default native and infrastructure +retries. Only the two coordinator-crash invocations shorten recovery to 60 seconds; +ordinary qualification and production runs use the 1200-second default. +The final qualification below establishes the planned and targeted paths, both +crash scenarios, temporary placement, diagnostics, and pool deletion. ## Validation record @@ -53,7 +57,7 @@ Slice 2 refinements 1 and 2 are implemented: reuse the timing-provider constant resolve a targeted Vitest config relative to its surface cwd. Recovery receipts carry the actual prior run/lock, worker, renewal/deadline and attributed process cleanup, rather than assuming a coordinator's OS death can be proven remotely. -The live receipt, retention and Chromium temporary-file observations remain open. +The final live qualification establishes receipt retention and Chromium temporary-file placement. Primitive's warning `dec_8e3d38a8` (check `td7crdek1b1n4dhajyww5p174s8h23bq`) remains unread because its CLI failed to install @@ -68,9 +72,9 @@ on both private workers. Its retained worker records and native reports are in helper resets `.testshards/results`, so initial identity/cleanup/receipt evidence must survive in attempt scratch and be restored after checkout, on shell failure, and by confirmed teardown after a killed shell. The correction adds that fixture -boundary without changing repository selection or checkout behavior. The new -leases remain held until their ordinary deadlines (about 19:31 UTC); no lock was -stolen or deadline edited. +boundary without changing repository selection or checkout behavior. Those +leases remained held until their ordinary deadlines (about 19:31 UTC); no lock was +stolen or deadline edited. The subsequent green runs recovered them normally. Read-only inventory at 19:29:39 UTC counted 146 sandboxes, all reported DEPLOYED, @@ -84,3 +88,224 @@ without increasing the private worker count or touching shared workers. lease run deliberately supplies `--lock-ttl-seconds 60` to qualify recovery from the retained failed attempt; native and infrastructure retry budgets stay at their defaults. The following contract run uses the ordinary recovery default. + +## Focused green checkpoint + +The checkout-reset correction passed the canonical runner at `c7482876`: +lease seam `.test-runs/2026-10-27T19-32-10-022Z-p22962-rfb177538dcaa4d24` +(seven cases, deliberate 60-second recovery interval), then contracts +`.test-runs/2026-10-27T19-32-20-996Z-p23059-r89282347b68d6c0c` +(nine cases, ordinary interval). Both completed on the first infrastructure +attempt with no test failures, recovered retries or leaks. Downloaded recovery +receipts survived the results reset, and the leases released after retrieval. +Lint and type checking passed at this revision and again at `49cfe0c3`. +The combined logs are in `.test-runs/slice-3-static/rose-slice3-green3.log` +and `rose-slice3-affected.log`. The earlier `53d800fa` type failure was a test +callback declaration requiring an unused SDK method; its log is +`rose-slice3-static3.log` in the same directory. + +At `49cfe0c3`, the seam and contracts passed again in +`2026-10-27T19-33-58-506Z-p23646-raaae2ad64b76b375` and +`2026-10-27T19-35-29-126Z-p23746-r01712028c5135ed1`. The provider file in +`2026-10-27T19-35-43-906Z-p23846-r3318f29853e73be5` exposed two source assertions +that still expected the old dispatch and interrupted-report wiring. The callback +behavior assertions passed. `720cef01` updated those two source assertions, +then ran the narrow pair before their whole file and the affected files below. +All listed runs used `npm test -- blaxel --pool rose-cleanup --file ` +with ordinary retry and recovery defaults; the first also selected the two names +with `--grep`. Full downloaded trees are under `.test-runs//`. + +| Revision | Selection | Canonical run | Result | +| --- | --- | --- | --- | +| `720cef01` | `test/repo/test-runner-provider.test.ts (two named cases)` | `2026-10-27T19-37-58-570Z-p24402-r4cdc7d5562c47eee` | passed; infrastructure completed | +| `720cef01` | `test/repo/test-runner-provider.test.ts` | `2026-10-27T19-38-09-966Z-p24499-r70f1f3eed2e449a5` | passed; infrastructure completed | +| `720cef01` | `test/repo/blaxel-pool-discovery.test.ts` | `2026-10-27T19-38-20-913Z-p24595-r84122d6e0b62dd4c` | passed; infrastructure completed | +| `720cef01` | `test/repo/blaxel-stale-owner-audit.test.ts` | `2026-10-27T19-39-40-343Z-p24692-rcb3f084f0f0c7c95` | passed; infrastructure completed | +| `720cef01` | `test/repo/test-runner-config.test.ts` | `2026-10-27T19-39-51-507Z-p24788-r2f67497f4e209dab` | passed; infrastructure completed | +| `720cef01` | `test/repo/resource-diagnostics.test.ts` | `2026-10-27T19-40-02-197Z-p24884-r776a0c6fb3273936` | passed; infrastructure completed | +| `720cef01` | `test/repo/resource-diagnostics-links.test.ts` | `2026-10-27T19-42-16-320Z-p24982-rf5458848d49b258a` | passed; infrastructure completed | +| `720cef01` | `test/repo/test-runner-resource-parent.test.ts` | `2026-10-27T19-42-28-510Z-p25079-r8d56b8721e4a08db` | passed; infrastructure completed | +| `720cef01` | `test/repo/test-runner-shard-worker.test.ts` | `2026-10-27T19-42-41-867Z-p25175-rf392193489fc6ad8` | passed; infrastructure completed | +| `720cef01` | `test/node/owned-process.e2e.test.ts` | `2026-10-27T19-43-03-198Z-p25283-r3d5b7ad0c998a6d0` | passed; infrastructure completed | + +Every run in this table has zero failed tests, recovered retries, process leaks and infrastructure failures. + +At `5b23a426`, the delayed-entry case additionally checks that an unregistered +request cannot release after teardown; the audit case verifies that available-worker +audit preparation/teardown preserves legacy residue and non-current scratch. +Each new assertion passed narrowly, followed by both complete files: + +| Selection | Canonical run | Result | +| --- | --- | --- | +| `test/repo/blaxel-worker-lease.test.ts (named case)` | `2026-10-27T19-43-45-072Z-p25796-r8622f05d670cea64` | passed; infrastructure completed | +| `test/repo/blaxel-lease-contracts.test.ts (named case)` | `2026-10-27T19-43-56-561Z-p25892-rd9b8c692ec75d21a` | passed; infrastructure completed | +| `test/repo/blaxel-worker-lease.test.ts` | `2026-10-27T19-44-07-741Z-p25987-rbb26c267c14b8a41` | passed; infrastructure completed | +| `test/repo/blaxel-lease-contracts.test.ts` | `2026-10-27T19-44-18-970Z-p26082-r36dc8f04d41f53f9` | passed; infrastructure completed | + +All four completed on the first infrastructure attempt, with no failures, recovered retries or leaks. + +## Live qualification and intermediate failures + +The first driver invocation at `5b23a426` stopped at coordinator preflight: +`2026-10-27T19-45-11-581Z-p26606-r3f51d29611472c9b` records missing `ss`. +Installed `iproute2` on `tv-cleanup-coordinator-rose` only; no worker changed. +The next canonical invocation uses the same pushed revision and normal retry +budget. Its first native attempt (`blaxel-live-EQZRnS`) found that Blaxel accepts +sandbox deletion before listing stops returning the deleted workers. The manager +had removed dispatch eligibility before its accepted deletes; fresh-only ensure +correctly refused the still-visible pool. Its final cleanup completed. SDK 0.3.24's +`SandboxInstance.delete` awaits the delete API response, not absence from listing. +The driver correction at `9d197e8f` polls for confirmed absence after a successful +canonical delete before recreating or claiming the pool gone. No lease protocol, +retry default, or provider configuration changes for this correction. + +The second native attempt (`blaxel-live-UjVPFN`) crossed planned/targeted retrieval, +competition and held deletion, actual TMPDIR placement, and both coordinator-kill +scenarios before reaching the immediate final-deletion listing check. Its retained +planned, targeted, active-recovered and quiet-recovered runs respectively are: + +- `2026-10-27T19-47-58-353Z-p27645-r7a1cb06d08f363aa` +- `2026-10-27T19-48-57-858Z-p27882-r0bc76a565de0ad6d` +- `2026-10-27T19-50-34-628Z-p28106-r6bde140566d56c43` +- `2026-10-27T19-51-51-273Z-p28338-r1db5ab714ba682ea` + +Each reports the intentional one failed fixture and one recovered native fixture, +completed infrastructure and no process leaks. The active receipt lists seven +and eight attributed processes on the two workers, including SIGKILL for surviving +Node processes. Quiet receipts have empty process lists. Failed-surface sidecars +are about 1.8 KiB; kernel access was available with no matching lines during the +observed intervals. That does not establish the absence of an OOM outside those +intervals. Chromium `148.0.7778.96` from +`/home/playwright/.cache/ms-playwright/chromium-1208/chrome-linux64/chrome` +held sixteen observed descriptors under the attempt TMPDIR, including deleted +`.org.chromium.Chromium.*` files. These are placement observations, not a memory +pressure reproduction or measured savings claim. This intermediate driver revision +is not the final green qualification. + +The canonical driver run at `5b23a426`, +`2026-10-27T19-46-15-645Z-p27336-r76172043306620d7`, completed infrastructure but +failed all three native attempts. Its attempt recorder shows 70.9 s, 357.7 s and +348.8 s. The first failed the fresh-only ensure assertion; the other two reached +the final delete then observed two workers still listed. No other assertion failed. +All three attempt directories (`blaxel-live-EQZRnS`, `blaxel-live-UjVPFN`, +`blaxel-live-wQ4JDh`) retain their logs and successful final cleanup records. +Both full attempts' child run trees and the outer native attempt report are +copied under `.test-runs/`. Default retries were allowed to finish; no revision +changed under this invocation. The corrected one-case driver runs at `9d197e8f`. + +At `9d197e8f`, the first driver attempt (`blaxel-live-pyyGvj`) reached a targeted +run that recovered an infrastructure failure: +`2026-10-27T20-01-14-534Z-p30344-r4b185c22b003ef58`. Attempt 1's remote entry and +teardown rejected a lost/expired 60-second lease before native evidence existed; +its exact renewal/transport timing was not retained and the cause of that lapse +is not established. Default infrastructure retry ran on the other worker and +retained the expected native failed/recovered outcomes under `attempt-2/`. +The driver still looked for `temporary-placement.json` outside that attempt +subdirectory. `d32457f7` derives artifact roots from the retained shard output +instead. It also corrects the live driver's overly broad 60-second override: +only deliberate abandoned-run invocations use it, as the plan requires; ordinary +retrieval and reuse use the production 1200-second default. No production lease +interval or retry budget changed. + +The `9d197e8f` outer run +`2026-10-27T20-00-04-460Z-p29961-r0fbeb800a4123ebe` passed on its second native +attempt (8m 39s total), followed by clean lint and type checking. Its first native +failure is exactly the placement-artifact existence assertion at line 163; its +second attempt (`blaxel-live-7Frvi5`) passed every live assertion, observed both +private workers disappear, and retained `cleanup.json` with exit 0. The outer +result correctly reports one recovered driver attempt. The complete outer report, +both live attempt directories and all child runs are retained. The static output +is `.test-runs/slice-3-static/rose-slice3-live3.log`. + +The `d32457f7` driver run, +`2026-10-27T20-09-47-242Z-p31959-reebae862cce797b7`, passed on its second native +attempt, then passed lint and type checking. The first failure was the driver's +120-second download-rendezvous wait in the quiet scenario, not a native fixture +assertion; both gate files eventually existed. The completed retry is +`blaxel-live-9zKaAr`, with passed completion and exit-0 cleanup, and all child +reports retained. The final driver allows 600 seconds to reach that rendezvous, +covering the runner's 480-second cold execution allowance plus teardown. This +changes neither a native retry budget nor a production worker timeout. Its static +log is `.test-runs/slice-3-static/rose-slice3-live4.log`. + +## Preparation without dispatch + +A final source pass found that `executionRequested: false` was being used as a +proxy for “no evidence produced.” Preparation can already have produced identity, +cleanup and recovery files before dispatch fails. At `06b07c0f`, the unchanged +retention implementation was extracted into the existing lease-lifecycle module +and a focused contract crossed real worker preparation/teardown and local writes, +substituting only the SDK read transport. Canonical run +`2026-10-27T20-25-30-716Z-p34389-r43c8aec32b92d34f` failed at release because the +prepared file had not been downloaded; infrastructure completed and the missing +file assertion was the intended red. + +`45202ee9` tracks whether preparation was requested and stops treating that state +as evidence-free merely because execution was not requested. Available artifacts +are also downloaded before propagating a required log-read failure, so missing +logs cannot suppress other retained evidence. Unknown preparation or retrieval +still holds the lease. No new worker state or recovery branch was added. + +The retention correction passed narrowly, then the full contracts, provider and +stale-owner files passed at `45202ee9`, with no native failures, recovered retries, +leaks or infrastructure failures. Lint and type checking also passed; the combined +log is `.test-runs/slice-3-static/rose-slice3-retention-green.log`. + +| Selection | Canonical run | +| --- | --- | +| `test/repo/blaxel-lease-contracts.test.ts (named case)` | `2026-10-27T20-28-25-923Z-p34956-r7a8e2fe5488f771c` | +| `test/repo/blaxel-lease-contracts.test.ts` | `2026-10-27T20-29-15-006Z-p35077-rc75cd93e2b864b85` | +| `test/repo/test-runner-provider.test.ts` | `2026-10-27T20-29-27-245Z-p35174-r098f4b0d3cd1cad7` | +| `test/repo/blaxel-stale-owner-audit.test.ts` | `2026-10-27T20-29-39-395Z-p35271-r3dc4727db3aa7390` | + +## Final qualification and handoff + +At `45202ee9`, `npm test -- local --surface experiment:blaxel-harness` on +`tv-cleanup-coordinator-rose` passed on its first native attempt in 271.207 seconds: +`2026-10-27T20-31-21-143Z-p35862-rde268122d6db9e34`. The outer driver reports zero +failed or recovered tests, process leaks and infrastructure failures. It kept the +ordinary native retry budget of two and infrastructure budget of two. Ordinary +child runs used the default 1200-second recovery interval; only the two deliberately +abandoned runs supplied `--lock-ttl-seconds 60`. + +The complete live attempt is `.test-runs/blaxel-live-YSZi2G/`. Its completion and +cleanup records show success and exit 0. Every pointer below resolves to a fully +copied local run tree under `.test-runs/`; the two deliberately killed coordinators +truthfully remain incomplete. The fixtures intentionally yield one failed native +case and one recovered case in each completed child run. These expected native +failures are assertions of the passing driver, not unaddressed suite failures. + +| Scenario | Canonical child run | Expected result | +| --- | --- | --- | +| planned transfer and retrieval | `2026-10-27T20-31-43-447Z-p36010-r102a02d53fa2b20e` | one deliberate native failure and one recovered case; infrastructure completed | +| targeted retrieval and reuse | `2026-10-27T20-32-52-337Z-p36243-r7d1d01b2f31bd9ae` | one deliberate native failure and one recovered case; infrastructure completed | +| held-capacity refusal | `2026-10-27T20-32-42-262Z-p36128-r7e9debd7b5412984` | incomplete; no passing selected surface | +| active coordinator killed | `2026-10-27T20-33-03-214Z-p36326-r50763ec34c56268e` | incomplete; no passing selected surface | +| active orphan recovered | `2026-10-27T20-34-11-856Z-p36450-rfdddc02ece515dce` | one deliberate native failure and one recovered case; infrastructure completed | +| quiet coordinator killed | `2026-10-27T20-34-25-068Z-p36548-r02e5eeb7df18d744` | incomplete; no passing selected surface | +| quiet orphan recovered | `2026-10-27T20-35-28-816Z-p36690-r7eb081f69c2f7b8b` | one deliberate native failure and one recovered case; infrastructure completed | +| dispatch refused after deletion | `2026-10-27T20-35-48-311Z-p36815-r3d7413d5032dbe88` | incomplete; no passing selected surface | + +The driver observed renewal while downloads were held, refusal of competing +acquisition and deletion, unchanged held reports, correct report/native-attempt +links after scratch removal, both known-prefix removals and stale-scratch removal, +and preservation of unrelated `/tmp` and persistent-cache sentinels. Real Node +paths and Chromium descriptors (including unlinked files) were under the shard's +TMPDIR. Active recovery signalled attributed processes; quiet recovery used the +same procedure with no survivors. Both incoming runs retained the old identity, +renewal/deadline, cleanup outcome and discarded-report receipts. + +The final pool manager deleted `tv-testshard-x64-rose-cleanup-01` and `-02`, waited +for confirmed absence, and a fresh canonical dispatch failed because the pool was +empty. The private coordinator remains allocated with its evidence for the later +integrated gate. No shared worker was used or changed during qualification. + +The implementation and proof citations are ready for independent review, not for +merge. Slice review, integrated-result review, target reconciliation, the ordinary +full Blaxel verification gate, pre-merge docs preparation and Josh's human spec/PR +review remain. This evidence establishes lifecycle behavior, not peak memory use, +measured suite-wide savings, a new exhaustion reproduction or provider latency +bounds. The earlier shortened-lease expiry's precise timing remains unknown and +is recorded above. No production behavior, default retry budget, browser version +or shard count was changed. Primitive's unread warning remains recorded above; +its CLI was not debugged further. diff --git a/proofs/arch/test-runner/blaxel-testshards.md b/proofs/arch/test-runner/blaxel-testshards.md index 13788a80..a07843c4 100644 --- a/proofs/arch/test-runner/blaxel-testshards.md +++ b/proofs/arch/test-runner/blaxel-testshards.md @@ -12,9 +12,9 @@ Lease decisions and failure ordering are checked cheaply at the worker/coordinat The live acceptances below enter through the canonical CLI on a dedicated pool: normal retrieval, refusal while another coordinator holds the worker, recovery after coordinator death, and pool creation/deletion. They share a small committed fixture selection and fresh workers rather than repeating a full suite for every failure permutation. Both planned and targeted execution paths are crossed. Resource retention composes with [reporting's diagnostics assertions](reporting.md#Resource diagnostics); helper teardown belongs to [the testing-policy proof](../testing-policy.md#Worker-owned builds). -The spec's [Testing](../../../specs/arch/test-runner/blaxel-testshards.md#Testing) directives require real API/transport, fresh provisioning, and observed Node/Chromium temporary paths. Private workers and bounded disposable processes supply those observations; no deliberately exhausted host or shared-pool contamination is needed. The live driver and its native fixtures live in `experiment:blaxel-harness` and `experiment:blaxel-harness-fixtures`, excluded from `all` and ordinary CI by their experiment kind. Mocks below forfeit those crossings to the named live assertions. New assertions are obligations, not evidence of runs already performed. +The spec's [Testing](../../../specs/arch/test-runner/blaxel-testshards.md#Testing) directives require real API/transport, fresh provisioning, and observed Node/Chromium temporary paths. Private workers and bounded disposable processes supply those observations; no deliberately exhausted host or shared-pool contamination is needed. The live driver and its native fixtures live in `experiment:blaxel-harness` and `experiment:blaxel-harness-fixtures`, excluded from `all` and ordinary CI by their experiment kind. Mocks below forfeit those crossings to the named live assertions. The live driver shares these observations within one fresh private pool; focused contracts carry uncommon failure permutations without repeating paid-service runs. -`test/repo/blaxel-stale-owner-audit.test.ts` checks diagnostic construction and coordinator ordering on every host. Its live diagnostic cases require Linux procfs and run only on Linux; they establish no macOS backend coverage. `test/repo/test-runner-provider.test.ts` exercises shell activation ordering with an authored nvm replacement in a shell isolated from host startup files. That contract check does not install nvm or establish real worker activation. +The lease contracts and filesystem seams run only on Linux because their production helpers inspect procfs and mount records. This does not establish macOS worker support. `test/repo/blaxel-stale-owner-audit.test.ts` checks diagnostic construction and coordinator ordering on every host. Its live diagnostic cases require Linux procfs and run only on Linux; they establish no macOS backend coverage. `test/repo/test-runner-provider.test.ts` exercises shell activation ordering with an authored nvm replacement in a shell isolated from host startup files. That contract check does not install nvm or establish real worker activation. Repository selection composes several pieces across clean seams. A contract test proves URL conversion. A seam test runs real Git against a temporary checkout's `origin` to prove selection, and a process-boundary test runs the real coordinator to prove that an unusable selection stops it before dispatch. A further seam test executes the worker's generated `checkout` script under real `bash` and Git against two local bare repositories with shared history, proving that one persistent checkout moves between repositories in both directions without re-cloning or deleting ignored dependency files. That test reaches the repositories through local paths, so it forfeits GitHub HTTPS transport and token authentication. The coordinator's use of the selector and the generated script is a source assertion. The live runs that the spec's [Testing](../../../specs/arch/test-runner/blaxel-testshards.md#Testing) directive orders for a repository-selection change carry the GitHub transport, token, and real worker handoff: they run from checkouts of two repositories on the same pool, including workers whose checkout last served the other repository. When the two runs' commits have identical dependency declarations, those workers' logs must show no clone and a `deps` cache hit, which is the live evidence that switching repositories keeps installed dependencies. A focused test runs the worker's generated `deps` script after real checkouts of both repositories, with a stand-in `npm`. It proves that the cache decision follows the dependency declarations and ignores which repository supplied them. @@ -35,13 +35,15 @@ The worker lock and temporary-cleanup helpers accept filesystem roots for dispos A coordinator milestone hook, enabled only by `TV_TEST_RUNNER_SELFTEST`, can wait on a fixture-controlled rendezvous immediately before report download while renewal continues; ordinary runs do not wait. It changes scheduling, not lease state, process results, or report bytes. Live recovery uses the existing `--lock-ttl-seconds` override to shorten the abandonment wait, not a fake clock or altered retry budget. +`retainAttemptEvidence` accepts the lease’s SDK filesystem and a coordinator download directory. The prepared-but-undispatched contract substitutes reads backed by real fixture files; worker inventory creation, teardown and local writes remain real. Production reads the worker results tree through the installed SDK. + Pool-management contract tests may supply recording/failing SDK operations; ordinary pool commands use the installed SDK. These substitutions forfeit provider behavior to the live pool acceptance below. ## Assertions ### Test assertions -- Pool discovery includes matching workers on later API pages and excludes workers with other project, purpose, pool, or architecture labels, as required by the spec's [Pool](../../../specs/arch/test-runner/blaxel-testshards.md#Pool) contract. **Contract:** the consumer is the shared discovery helper and installed SDK, called in a child process; the service is the HTTP mock described above. The helper collects the unfiltered listing before filtering locally. Both manager and coordinator modes must traverse all pages, including an earlier page containing no matching workers. The coordinator sees only published, matching `DEPLOYED` workers; the manager also retains the pool's unpublished and partially provisioned workers. A page-fetch failure rejects discovery rather than returning a partial pool. Fixture sandbox records omit a region so discovery needs no data-plane connection. (policy-grade test for existing pagination: `test/repo/blaxel-pool-discovery.test.ts`; publication/partial-provisioning extension: test to be written) ^blaxel-pool-pagination +- Pool discovery includes matching workers on later API pages and excludes workers with other project, purpose, pool, or architecture labels, as required by the spec's [Pool](../../../specs/arch/test-runner/blaxel-testshards.md#Pool) contract. **Contract:** the consumer is the shared discovery helper and installed SDK, called in a child process; the service is the HTTP mock described above. The helper collects the unfiltered listing before filtering locally. Both manager and coordinator modes must traverse all pages, including an earlier page containing no matching workers. The coordinator sees only published, matching `DEPLOYED` workers; the manager also retains the pool's unpublished and partially provisioned workers. A page-fetch failure rejects discovery rather than returning a partial pool. Fixture sandbox records omit a region so discovery needs no data-plane connection. (policy-grade test: `test/repo/blaxel-pool-discovery.test.ts`, including unpublished and partially provisioned worker records) ^blaxel-pool-pagination The repository-selection assertions: @@ -67,30 +69,30 @@ The remaining coverage lives in `test/repo/test-runner-provider.test.ts`. - Ordinary cleanup/release coverage is superseded by [the lease ordering contract](#^blaxel-t-lease-order) and [the live retrieval acceptance](#^blaxel-t-retrieval); shell completion alone no longer permits release. - A stale-owner setup failure records the contaminated sandbox and its lifecycle evidence, excludes that sandbox from subsequent attempts, and permits a configured infrastructure retry only on a distinct sandbox that independently passes lease acquisition and owner audit. -**Boundary gaps — stated honestly per [testing-policy.md](../../../specs/arch/testing-policy.md):** the classifier, normalization, bounded acquisition scheduler, and provisioning/activation/cache command shapes are covered without calling Blaxel. The scheduler test uses an injected acquisition callback and therefore does not cross the Blaxel API boundary. Code-only checks cannot prove installation or operation inside the base image. The live assertions below must supply that evidence; their pending citations must not be read as a passed provider run. Repository switching on real workers has no committed test; the live runs from two repositories' checkouts described in the coverage model remain required for a repository-selection change. +**Boundary gaps — stated honestly per [testing-policy.md](../../../specs/arch/testing-policy.md):** the classifier, normalization, bounded acquisition scheduler, and provisioning/activation/cache command shapes are covered without calling Blaxel. The scheduler test uses an injected acquisition callback and therefore does not cross the Blaxel API boundary. Code-only checks cannot prove installation or operation inside the base image. The live assertions below supply that provider evidence. Repository switching on real workers has no committed test; the live runs from two repositories' checkouts described in the coverage model remain required for a repository-selection change. ### Lease, recovery, and scratch -- **Contract — lease decisions.** The shared lock decision layer consumes authored current, expired, legacy, incomplete, and unreadable records with a supplied worker clock and audit/signal results. Under [Locking](../../../specs/arch/test-runner/blaxel-testshards.md#Locking) and [recovery](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery), renewal preserves ownership only for the matching unexpired lease; the default recovery interval is 1200 seconds regardless of execution timeout. Expiry permits cleanup, not dispatch: both quiet and active abandoned runs reach a new lease only after confirmed cleanup and a clean audit. Unknown identity or surviving owners hold the worker. Incomplete/unreadable records have the same clean-audit requirement for acquisition and deletion. Legacy positive TTLs are respected, live legacy owners are skipped, and new records use `expiresAt: 0`. Audit mode never recovers or deletes residue. Mock clock and audit/signal operations forfeit real serialization and termination to the guard seam and live recovery below. (test to be written) ^blaxel-t-lease-decisions +- **Contract — lease decisions.** The shared lock decision layer consumes authored current, expired, legacy, incomplete, and unreadable records with a supplied worker clock and audit/signal results. Under [Locking](../../../specs/arch/test-runner/blaxel-testshards.md#Locking) and [recovery](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery), renewal preserves ownership only for the matching unexpired lease; the default recovery interval is 1200 seconds regardless of execution timeout. Expiry permits cleanup, not dispatch: both quiet and active abandoned runs reach a new lease only after confirmed cleanup and a clean audit. Unknown identity or surviving owners hold the worker. Incomplete/unreadable records have the same clean-audit requirement for acquisition and deletion. Legacy positive TTLs are respected, live legacy owners are skipped, and new records use `expiresAt: 0`. Audit mode never recovers or deletes residue. Mock clock and audit/signal operations forfeit real serialization and termination to the guard seam and live recovery below. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “expired active and quiet leases use the same cleanup; uncertain owners hold”, “legacy, incomplete, unreadable and audit leases fail closed without clean ownership”, and “cleanup failures keep the old identity, while explicit deletion reserves an expired worker”; `test/repo/blaxel-worker-lease.test.ts` renewable-default and old-identity cases) ^blaxel-t-lease-decisions -- **Contract — coordinator ordering and failures.** The coordinator's lease lifecycle uses recording/failing transport, download, and cleanup callbacks. Under [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release), renewal spans acquisition, execution, cleanup, and retrieval. A successful shell exit still requires descendant confirmation; unknown dispatch or failed cleanup cannot release. Successful retrieval and identity validation precede release; a download retry neither reruns tests nor resets worker results. An unused lease waits for pending writes and retains produced evidence. On interruption, the authoritative interrupted report precedes cleanup/retrieval and cannot become an ordinary completion report; cleanup still runs after a report-write error. A coordinator that loses ownership stops issuing mutations and reports infrastructure failure. These callbacks forfeit API, signals, and downloads to the live retrieval/recovery assertions; existing signal-report fixtures supply breadth without another native suite. (test to be written) ^blaxel-t-lease-order +- **Contract — coordinator ordering and failures.** The coordinator's lease lifecycle uses recording/failing transport, download, and cleanup callbacks. Under [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release), renewal spans acquisition, execution, cleanup, and retrieval. A successful shell exit still requires descendant confirmation; unknown dispatch or failed cleanup cannot release. Successful retrieval and identity validation precede release; a download retry neither reruns tests nor resets worker results. An unused lease waits for pending writes and retains produced evidence. On interruption, the authoritative interrupted report precedes cleanup/retrieval and cannot become an ordinary completion report; cleanup still runs after a report-write error. A coordinator that loses ownership stops issuing mutations and reports infrastructure failure. These callbacks forfeit API, signals, and downloads to the live retrieval/recovery assertions; existing signal-report fixtures supply breadth without another native suite. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “retrieval retries after teardown without rerunning work; uncertainty never releases”, “renewal continues while retrieval waits and stops on ownership loss”, and “preparation evidence is downloaded before an undispatched attempt releases”; `test/repo/test-runner-provider.test.ts` interrupted-report and explicit TTL-forwarding cases; `test/repo/blaxel-worker-lease.test.ts` delayed-entry case) ^blaxel-t-lease-order -- **Seam — guarded registration and ownership.** Production worker-entry and lock helpers cross real Linux process/filesystem boundaries in a private fixture root as an ordinary repository test on Linux. It requires no Blaxel API or credentials; it runs only on Linux because it uses procfs, signals and the OS guard. Two competing acquisitions yield only one owner; killing a process holding the short guard releases the OS guard without deleting another lease. A request paused before entry registration, then resumed after ownership changes, cannot touch a checkout sentinel. Old-ID renewal/release cannot change the new record. Registered setup descendants preserve the outer token when a nested surface replaces its own token; recovery stops only the attributed PID/start identities, including a descendant that ignores TERM and needs KILL. A separate live fixture owner remains untouched and prevents reuse. The fixtures are small processes, records produced by the real helper, and rendezvous files. The supplied audit filters real procfs observations to the fixture tokens, a mock of whole-worker scope that forfeits that protection to live recovery; process inspection, identity revalidation, signals and filesystem operations remain real. This proves [registration](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-reservation) and [recovery](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery), not provider dispatch. (test to be written) ^blaxel-t-worker-guard +- **Seam — guarded registration and ownership.** Production worker-entry and lock helpers cross real Linux process/filesystem boundaries in a private fixture root as an ordinary repository test on Linux. It requires no Blaxel API or credentials; it runs only on Linux because it uses procfs, signals and the OS guard. Two competing acquisitions yield only one owner; killing a process holding the short guard releases the OS guard without deleting another lease. A request paused before entry registration, then resumed after ownership changes, cannot touch a checkout sentinel. Old-ID renewal/release cannot change the new record. Registered setup descendants preserve the outer token when a nested surface replaces its own token; recovery stops only the attributed PID/start identities, including a descendant that ignores TERM and needs KILL. A separate live fixture owner remains untouched and prevents reuse. The fixtures are small processes, records produced by the real helper, and rendezvous files. The supplied audit filters real procfs observations to the fixture tokens, a mock of whole-worker scope that forfeits that protection to live recovery; process inspection, identity revalidation, signals and filesystem operations remain real. This proves [registration](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-reservation) and [recovery](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery), not provider dispatch. (policy-grade tests: `test/repo/blaxel-worker-lease.test.ts` competing acquisition, delayed entry, old-identity and registered-descendant recovery cases) ^blaxel-t-worker-guard -- **Seam — bounded filesystem cleanup.** The production cleaner crosses the real filesystem in a disposable parent containing both allowed legacy prefixes, stale scratch with complete and missing metadata, the current scratch, unrelated files/directories, and symlinks to outside sentinels. With a lease and clean-audit precondition supplied as a mock, it removes exactly the permitted real directories, leaves the current directory and outside sentinels intact, rejects symlink roots, and records counts, allocated bytes and free-space observations under [known leftovers](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-known-leftovers). The audit mock forfeits real disuse confirmation to the live retrieval/recovery cases. No assertion equates allocated bytes with the filesystem's free-space increase. (test to be written) ^blaxel-t-scratch-cleanup +- **Seam — bounded filesystem cleanup.** The production cleaner crosses the real filesystem in a disposable parent containing both allowed legacy prefixes, stale scratch with complete and missing metadata, the current scratch, unrelated files/directories, and symlinks to outside sentinels. With a lease and clean-audit precondition supplied as a mock, it removes exactly the permitted real directories, leaves the current directory and outside sentinels intact, rejects symlink roots, and records counts, allocated bytes and free-space observations under [known leftovers](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-known-leftovers). The audit mock forfeits real disuse confirmation to the live retrieval/recovery cases. No assertion equates allocated bytes with the filesystem's free-space increase. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “only known real directories are removed, with allocated-byte evidence and outside links preserved”; `test/repo/blaxel-worker-lease.test.ts` “attempt identity and recovery evidence survive checkout resetting the result tree”) ^blaxel-t-scratch-cleanup -- **Contract — cleanup uncertainty.** The cleaner's decision layer consumes authored filesystem-probe results and removal errors. An unexpected mount boundary is skipped, and an incomplete removal is reported without claiming the whole directory was reclaimed; neither permits new test dispatch under [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#Temporary files). These mocked conditions forfeit actual filesystem operations to the cleanup seam and live retrieval assertion; a real mount or full disk adds no necessary coverage. (test to be written) ^blaxel-t-scratch-errors +- **Contract — cleanup uncertainty.** The cleaner's decision layer consumes authored filesystem-probe results and removal errors. An unexpected mount boundary is skipped, and an incomplete removal is reported without claiming the whole directory was reclaimed; neither permits new test dispatch under [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#Temporary files). These mocked conditions forfeit actual filesystem operations to the cleanup seam and live retrieval assertion; a real mount or full disk adds no necessary coverage. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “mount boundaries and incomplete removal remain reported failures without reclaimed counts” and “cleanup failures keep the old identity, while explicit deletion reserves an expired worker”) ^blaxel-t-scratch-errors Shared live declaration: the following acceptances run in `experiment:blaxel-harness` via `npm test -- local --surface experiment:blaxel-harness` on a dedicated credentialed Blaxel coordinator sandbox outside the worker pool, from a committed and pushed checkout, never a shared developer host or a leased worker, under [harness placement](../../../specs/arch/test-runner/test-runner.md#^blaxel-harness-placement). They spawn the canonical CLI with real argv and inspect its exit and retained run directory. A fresh dedicated pool, at least two committed native fixture files in `experiment:blaxel-harness-fixtures`, known leftover directories and unrelated sentinels are fixtures. The two files permit the planned run's file-atomic two-shard assignment. Under [surface selection](../../../specs/arch/test-runner/test-registry.md#^blaxel-harness-surfaces), `npm test -- blaxel --tag blaxel-harness-fixture --pool --shards 2 --allow-shard-count-override 1` delegates `--suite selection --surfaces experiment:blaxel-harness-fixtures`; `npm test -- blaxel --surface experiment:blaxel-harness-fixtures --pool ` takes the targeted path. Workers read the committed registry. The coordinator, SDK, worker scripts, native runners, transport, ownership audits and filesystem operations are real. The report-download rendezvous above is the only scheduling hook; no mechanism is mocked. A bounded test driver owns and finally cleans up its coordinators and private workers. These tests do not invoke fixtures restricted to isolated GitHub hosts. -- **Acceptance — reports before reuse.** A small planned run with at least two shards traverses real acquisition, architecture verification, checkout, runtime/dependency setup, execution and log/report transport under [the shard lifecycle](../../../specs/arch/test-runner/blaxel-testshards.md#Shard lifecycle). Each uploaded script path lies inside its current shard attempt directory. The retained setup phases include `temporary-cleanup`, with cleanup counts and free-space evidence. While it waits at the download rendezvous, observed renewals keep the leases valid; a competing canonical run cannot acquire the occupied capacity or alter its result files. After retrieval, the original native reports, setup/cleanup evidence and diagnostics have the original identities and readable local paths, scratch is gone, and the workers become available to a targeted run. Legacy-prefix and stale-scratch fixtures disappear, unrelated sentinels and persistent caches survive, and the next run does not accumulate an older results tree. The targeted run crosses its distinct dispatch path too. This proves [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release) and [scratch lifetime](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-scratch); the shared diagnostics observations are owned by [reporting](reporting.md#^reporting-t-resource-live). (test to be written) ^blaxel-t-retrieval +- **Acceptance — reports before reuse.** A small planned run with at least two shards traverses real acquisition, architecture verification, checkout, runtime/dependency setup, execution and log/report transport under [the shard lifecycle](../../../specs/arch/test-runner/blaxel-testshards.md#Shard lifecycle). Each uploaded script path lies inside its current shard attempt directory. The retained setup phases include `temporary-cleanup`, with cleanup counts and free-space evidence. While it waits at the download rendezvous, observed renewals keep the leases valid; a competing canonical run cannot acquire the occupied capacity or alter its result files. After retrieval, the original native reports, setup/cleanup evidence and diagnostics have the original identities and readable local paths, scratch is gone, and the workers become available to a targeted run. Legacy-prefix and stale-scratch fixtures disappear, unrelated sentinels and persistent caches survive, and the next run does not accumulate an older results tree. The targeted run crosses its distinct dispatch path too. This proves [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release) and [scratch lifetime](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-scratch); the shared diagnostics observations are owned by [reporting](reporting.md#^reporting-t-resource-live). (policy-grade test: `test/runner-fixtures/blaxel-harness/driver/harness.test.ts` “qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle”, with `fixtures/outcomes.test.ts` and `fixtures/temporary.test.ts` under the same harness directory) ^blaxel-t-retrieval -- **Acceptance — abandoned run recovery.** In two small scenarios, kill the real coordinator: once while a shard supervisor and a token-bearing child remain, and once after the shard stops but before download. The next canonical run, after the shortened configured recovery interval, stops the old owned processes where present, confirms a clean worker, clears allowed residue, and runs on that worker. Its downloaded recovery receipt retains the old identity, renewal/deadline, cleanup result and discarded-report flag across results reset. No old report becomes a pass. This proves the same [automatic recovery path](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery) works for active and quiet orphans without waiting for their voluntary exit. Failure permutations are carried by the contracts, not more live crash runs. (test to be written) ^blaxel-t-recovery-live +- **Acceptance — abandoned run recovery.** In two small scenarios, kill the real coordinator: once while a shard supervisor and a token-bearing child remain, and once after the shard stops but before download. The next canonical run, after the shortened configured recovery interval, stops the old owned processes where present, confirms a clean worker, clears allowed residue, and runs on that worker. Its downloaded recovery receipt retains the old identity, renewal/deadline, cleanup result and discarded-report flag across results reset. No old report becomes a pass. This proves the same [automatic recovery path](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery) works for active and quiet orphans without waiting for their voluntary exit. Failure permutations are carried by the contracts, not more live crash runs. (policy-grade test: `test/runner-fixtures/blaxel-harness/driver/harness.test.ts` “qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle”, with `fixtures/outcomes.test.ts` and `fixtures/temporary.test.ts` under the same harness directory) ^blaxel-t-recovery-live -- **Seam — temporary-directory inheritance.** In the live planned and targeted fixture runs above, the shard environment reaches real Node and Chromium: Node's `os.tmpdir()` and created files are inside the current `tmp/`, and actual Chromium shared-memory descriptor paths resolve there, including unlinked files. Inspect the launched browser's descriptors on the worker; an environment-variable echo alone is insufficient. Nested surface tokens must not erase the outer ownership token or `TMPDIR`. After process teardown, current scratch is absent while retained sidecars are readable. This honors [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-scratch) and [Testing](../../../specs/arch/test-runner/blaxel-testshards.md#Testing). A small HTML page is the browser fixture; no browser or path mechanism is mocked, and no memory-pressure or exact byte-saving claim is made. (test to be written) ^blaxel-t-tmpdir-live +- **Seam — temporary-directory inheritance.** In the live planned and targeted fixture runs above, the shard environment reaches real Node and Chromium: Node's `os.tmpdir()` and created files are inside the current `tmp/`, and actual Chromium shared-memory descriptor paths resolve there, including unlinked files. Inspect the launched browser's descriptors on the worker; an environment-variable echo alone is insufficient. Nested surface tokens must not erase the outer ownership token or `TMPDIR`. After process teardown, current scratch is absent while retained sidecars are readable. This honors [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-scratch) and [Testing](../../../specs/arch/test-runner/blaxel-testshards.md#Testing). A small HTML page is the browser fixture; no browser or path mechanism is mocked, and no memory-pressure or exact byte-saving claim is made. (policy-grade test: `test/runner-fixtures/blaxel-harness/driver/harness.test.ts` “qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle”, with `fixtures/outcomes.test.ts` and `fixtures/temporary.test.ts` under the same harness directory) ^blaxel-t-tmpdir-live ### Pool mutation -- **Contract — publication and deletion ordering.** The pool manager uses authored sandbox states and recording/failing SDK operations. Under [Pool](../../../specs/arch/test-runner/blaxel-testshards.md#Pool) and [pool operations](../../../specs/arch/test-runner/blaxel-testshards.md#Older runners and pool operations), it publishes eligibility only after successful provisioning/verification, refuses to refill partial pools, refuses deletion of valid leases, and unpublishes before deleting an idle or expired worker. An expired lease can be deleted despite failed process cleanup; a failed deletion stays unavailable and visible. Unreadable records follow the common contract above; command failure on a previously eligible worker holds/reports it, while a fresh worker never published may be deleted without a lease. SDK substitutions forfeit actual metadata changes and deletion to the live pool acceptance. (test to be written) ^blaxel-t-pool-order +- **Contract — publication and deletion ordering.** The pool manager uses authored sandbox states and recording/failing SDK operations. Under [Pool](../../../specs/arch/test-runner/blaxel-testshards.md#Pool) and [pool operations](../../../specs/arch/test-runner/blaxel-testshards.md#Older runners and pool operations), it publishes eligibility only after successful provisioning/verification, refuses to refill partial pools, refuses deletion of valid leases, and unpublishes before deleting an idle or expired worker. An expired lease can be deleted despite failed process cleanup; a failed deletion stays unavailable and visible. Unreadable records follow the common contract above; command failure on a previously eligible worker holds/reports it, while a fresh worker never published may be deleted without a lease. SDK substitutions forfeit actual metadata changes and deletion to the live pool acceptance. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “pool eligibility follows verification, and deletion reserves then unpublishes” and “failed deletion of never-published workers remains retryable without their runtime”) ^blaxel-t-pool-order -- **Acceptance — fresh pool and safe deletion.** Through canonical pool commands and the real API, create the dedicated pool used by the live assertions, observe workers become eligible only after runtime/architecture verification, and run the fixtures on them. Attempt deletion while the retrieval gate holds valid leases: the command reports refusals and exits nonzero, and the original run still completes. After release, delete the pool and confirm its workers are absent; a new coordinator cannot dispatch there. This proves the [pool mutation](../../../specs/arch/test-runner/blaxel-testshards.md#Older runners and pool operations) and [operations](../../../specs/arch/test-runner/blaxel-testshards.md#Pool provisioning) spine through actual Blaxel label changes and deletion. These observations share the retrieval scenario rather than provisioning another pool. (test to be written) ^blaxel-t-pool-live +- **Acceptance — fresh pool and safe deletion.** Through canonical pool commands and the real API, create the dedicated pool used by the live assertions, observe workers become eligible only after runtime/architecture verification, and run the fixtures on them. Attempt deletion while the retrieval gate holds valid leases: the command reports refusals and exits nonzero, and the original run still completes. After release, delete the pool and confirm its workers are absent; a new coordinator cannot dispatch there. This proves the [pool mutation](../../../specs/arch/test-runner/blaxel-testshards.md#Older runners and pool operations) and [operations](../../../specs/arch/test-runner/blaxel-testshards.md#Pool provisioning) spine through actual Blaxel label changes and deletion. These observations share the retrieval scenario rather than provisioning another pool. (policy-grade test: `test/runner-fixtures/blaxel-harness/driver/harness.test.ts` “qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle”, with `fixtures/outcomes.test.ts` and `fixtures/temporary.test.ts` under the same harness directory) ^blaxel-t-pool-live diff --git a/proofs/arch/test-runner/reporting.md b/proofs/arch/test-runner/reporting.md index 929f59ca..2ec31c9f 100644 --- a/proofs/arch/test-runner/reporting.md +++ b/proofs/arch/test-runner/reporting.md @@ -10,7 +10,7 @@ Native-report contracts carry normalization breadth; real native-runner and CLI Resource diagnostics compose a bounded writer contract, a real Linux probe/parent-lifecycle seam, and the dedicated Blaxel acceptances owned by [the provider proof](blaxel-testshards.md#Lease, recovery, and scratch). A small native failure and a recovered retry establish retained evidence; they do not reproduce resource exhaustion. Authored unavailable/reset/oversized probe inputs cover uncommon diagnostic failures without filling a disk or causing an OOM kill. Those mocks forfeit kernel reading and transport to the real seams. -Planned and targeted execution share the collector; the parent seams cover capture and the dedicated Blaxel acceptances cover download. The same sidecars pass through provider normalization and attempt history; a recovered infrastructure attempt must not replace earlier evidence. Recovery receipts are observed in the provider's crash acceptance, not a duplicate crash here. No test claims start/end samples measure a peak or that an unavailable kernel log excludes OOM. The bounds, parent capture and retained-link assertions have passed on Blaxel; the dedicated live download/reuse and recovery-receipt acceptance remains pending in slice 3. +Planned and targeted execution share the collector; the parent seams cover capture and the dedicated Blaxel acceptances cover download. The same sidecars pass through provider normalization and attempt history; a recovered infrastructure attempt must not replace earlier evidence. Recovery receipts are observed in the provider's crash acceptance, not a duplicate crash here. No test claims start/end samples measure a peak or that an unavailable kernel log excludes OOM. The dedicated live acceptance checks download, reuse and recovery receipts alongside the focused bounds, parent capture and retained-link assertions. ## Test hooks @@ -54,4 +54,4 @@ The event-normalization assertions are **contract tests** over recorded native-r - **Contract — retained links and attempt history.** Provider normalization and infrastructure-attempt aggregation consume authored shard/task envelopes referring to real temporary sidecar files, with recorded native failed/recovered fixtures. Under [normalized results](../../../specs/arch/test-runner/reporting.md#Normalized surface result) and [resource diagnostics](../../../specs/arch/test-runner/reporting.md#Resource diagnostics), each advertised `resourceDiagnosticsPaths` entry exists relative to the final run directory; earlier attempt evidence survives a later successful infrastructure attempt. Missing native output remains incomplete and cannot manufacture a selected-surface result. Samples and kernel excerpts appear in neither compact console output nor timing events. Mocked provider envelopes forfeit remote creation/download to the live assertion; recorded native inputs forfeit attempt capture to the real parent seam and existing native-recorder assertions above. ([`test/repo/resource-diagnostics-links.test.ts`](../../../test/repo/resource-diagnostics-links.test.ts)) ^reporting-t-resource-links -- **Acceptance — evidence reaches the caller.** Share the dedicated-pool canonical CLI runs in [the provider retrieval acceptance](blaxel-testshards.md#^blaxel-t-retrieval), under its shared `experiment:blaxel-harness` declaration and dedicated coordinator placement. Its `experiment:blaxel-harness-fixtures` selection includes a committed fail-once native fixture that recovers under the default retry budget and a fixture that fails all attempts. Planned and targeted runs retain the native outcomes and their start/end sidecars after worker scratch is removed; every normalized diagnostic link opens a downloaded file with the correct run/shard/surface identity and native-attempt references. The failed run still fails, the recovered case retains both attempts, and any inaccessible kernel evidence says so. The provider's [crash acceptance](blaxel-testshards.md#^blaxel-t-recovery-live) supplies the downloaded recovery receipt and its disclosed report loss. Boundary: real CLI, Blaxel API, native runners and report files, with the provider proof's scheduling hook and no mocks. This proves [resource retention](../../../specs/arch/test-runner/reporting.md#Resource diagnostics) end to end without a resource-exhaustion experiment. (test to be written) ^reporting-t-resource-live +- **Acceptance — evidence reaches the caller.** Share the dedicated-pool canonical CLI runs in [the provider retrieval acceptance](blaxel-testshards.md#^blaxel-t-retrieval), under its shared `experiment:blaxel-harness` declaration and dedicated coordinator placement. Its `experiment:blaxel-harness-fixtures` selection includes a committed fail-once native fixture that recovers under the default retry budget and a fixture that fails all attempts. Planned and targeted runs retain the native outcomes and their start/end sidecars after worker scratch is removed; every normalized diagnostic link opens a downloaded file with the correct run/shard/surface identity and native-attempt references. The failed run still fails, the recovered case retains both attempts, and any inaccessible kernel evidence says so. The provider's [crash acceptance](blaxel-testshards.md#^blaxel-t-recovery-live) supplies the downloaded recovery receipt and its disclosed report loss. Boundary: real CLI, Blaxel API, native runners and report files, with the provider proof's scheduling hook and no mocks. This proves [resource retention](../../../specs/arch/test-runner/reporting.md#Resource diagnostics) end to end without a resource-exhaustion experiment. (policy-grade test: `test/runner-fixtures/blaxel-harness/driver/harness.test.ts` “qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle”, using the harness’s real failed/recovered native fixtures) ^reporting-t-resource-live diff --git a/proofs/arch/test-runner/test-registry.md b/proofs/arch/test-runner/test-registry.md index 2675d3d7..5a7edb9f 100644 --- a/proofs/arch/test-runner/test-registry.md +++ b/proofs/arch/test-runner/test-registry.md @@ -32,7 +32,7 @@ These run in-process as pure functions over the loaded registry, plus real files - `validateRegistry` ignores a runner config placed outside the canonical include roots (e.g. under `prototypes/` or `experiments/`). - The `all` suite resolves to the unit and non-agent e2e execution groups, in `order`, includes `unit:build-config`, and the `unit:workspaces` group contains the package unit surfaces in declared order. - Every lifecycle fault surface retains `kind: "experiment"` and `tag:isolated-github-only`; none is selected by `all`. -- **Contract — live harness selection.** Load the committed registry with the real selector and inspect the committed driver's dispatch selections. Both [Blaxel harness surfaces](../../../specs/arch/test-runner/test-registry.md#^blaxel-harness-surfaces) have `kind: "experiment"`; `all` and `e2e-ci` select neither, and tag `blaxel-harness-fixture` selects only `experiment:blaxel-harness-fixtures`. The driver's planned and targeted selections name only that fixture surface, never the driver, as required by [harness placement](../../../specs/arch/test-runner/test-runner.md#^blaxel-harness-placement). This is a source-level contract with no mocks or provider calls; real dispatch is carried by [the provider retrieval acceptance](blaxel-testshards.md#^blaxel-t-retrieval). (test to be written) ^registry-t-blaxel-harness-selection +- **Contract — live harness selection.** Load the committed registry with the real selector and inspect the committed driver's dispatch selections. Both [Blaxel harness surfaces](../../../specs/arch/test-runner/test-registry.md#^blaxel-harness-surfaces) have `kind: "experiment"`; `all` and `e2e-ci` select neither, and tag `blaxel-harness-fixture` selects only `experiment:blaxel-harness-fixtures`. The driver's planned and targeted selections name only that fixture surface, never the driver, as required by [harness placement](../../../specs/arch/test-runner/test-runner.md#^blaxel-harness-placement). This is a source-level contract with no mocks or provider calls; real dispatch is carried by [the provider retrieval acceptance](blaxel-testshards.md#^blaxel-t-retrieval). (policy-grade test: `test/repo/test-runner-config.test.ts` “keeps Blaxel harness fixtures separate from the driver and ordinary suites”) ^registry-t-blaxel-harness-selection - The `unit:build-config` surface owns `test/repo/build-config-integrity.test.ts`; `unit:root` excludes that file. - The `telemetry-posthog-roundtrip` suite resolves to the `telemetry-posthog-roundtrip:integration` surface, and that surface owns `packages/server/test/telemetry-posthog.integration.test.ts`; `unit:server` excludes that file. - The `daemon-acceptance` suite resolves to `daemon-acceptance:cli`, and that surface exclusively owns `test/node/daemon-acceptance.test.ts`; `e2e:node` excludes that file. The surface declares the `daemon-test-host` preflight and is absent from `all`. diff --git a/proofs/arch/test-runner/test-runner.md b/proofs/arch/test-runner/test-runner.md index cfd256d1..ac075999 100644 --- a/proofs/arch/test-runner/test-runner.md +++ b/proofs/arch/test-runner/test-runner.md @@ -10,7 +10,7 @@ Coverage declarations are carried inside the migrated assertion blocks below. Pr The [combined-project seam](sharded-execution.md#^combined-vitest-projects) crosses the shared `vitestProjectsArgs` handoff that writes the combined config and selects it for the Vitest command used by local and sharded runs. It exercises project roots, exact file selection and attempt recording. The local wrapper's pre-commands, execution without an explicit file list and result splitting remain within the real local end-to-end boundary gap below; the shared-helper seam does not claim that whole path. -The Blaxel exception to ordinary live-owner protection in [Surface process lifecycle](../../../specs/arch/test-runner/test-runner.md#Surface process lifecycle) is proven by [the worker guard seam](blaxel-testshards.md#^blaxel-t-worker-guard) and [automatic recovery acceptance](blaxel-testshards.md#^blaxel-t-recovery-live). The same provider proof carries preservation of the outer shard attempt token through nested surface scopes. These pending assertions do not weaken or relocate the existing ordinary-owner cleanup tests, including fixtures restricted to isolated GitHub execution. +The Blaxel exception to ordinary live-owner protection in [Surface process lifecycle](../../../specs/arch/test-runner/test-runner.md#Surface process lifecycle) is proven by [the worker guard seam](blaxel-testshards.md#^blaxel-t-worker-guard) and [automatic recovery acceptance](blaxel-testshards.md#^blaxel-t-recovery-live). The same provider proof carries preservation of the outer shard attempt token through nested surface scopes. These assertions do not weaken or relocate the existing ordinary-owner cleanup tests, including fixtures restricted to isolated GitHub execution. Two test-infrastructure rules concern the product's `tv serve` and are proven where the product is. The recipe for a test that spawns `tv serve` gives the server a temporary [Television home](../../../specs/product/cli.md#^cli-home) whose config file sets port `0`, passes that home with `--home` to the server and every client command, and passes the acquired port as client `--port`. It is a way of writing tests, honored by the design of the tests that spawn the server: the [product CLI proof](../../product/cli.md#Acceptance criteria: CLI spine) declares it for its acceptances, and review checks it elsewhere. No separate assertion is ordered for it; the repository scan below guards only numeric test-service URLs. The daemon-acceptance surface's form of installation (a temporary home, a released stable port written with `tv config set port`, then `tv --home serve --persist`) is carried by the same proof's [home-based persisted-install assertion](../../product/cli.md#^cli-ac-persist-home-install). From 6d5055978ec9744cb43deafdcd831aa1ac110b7e Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:03:49 +0000 Subject: [PATCH 51/64] Cover configurable Blaxel harness placement and recovery defaults --- ...ness-resource-cleanup-slice-3-review-r1.md | 70 +++++++++++++++++++ proofs/arch/test-runner/test-runner.md | 1 + test/repo/test-runner-guardrails.test.ts | 31 ++++++++ test/repo/test-runner-provider.test.ts | 4 ++ 4 files changed, 106 insertions(+) create mode 100644 docs/working/test-harness-resource-cleanup-slice-3-review-r1.md diff --git a/docs/working/test-harness-resource-cleanup-slice-3-review-r1.md b/docs/working/test-harness-resource-cleanup-slice-3-review-r1.md new file mode 100644 index 00000000..53859c45 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-slice-3-review-r1.md @@ -0,0 +1,70 @@ +# Slice 3 implementation review, round 1: test-harness resource cleanup (PR #12, 2dc0e7d6) + +**Verdict: FAIL, one blocker.** The lifecycle itself is right and the evidence for it is real. One worker is held from acquisition through confirmed teardown and retained results; only the coordinator renews, every worker-local change runs under one short `flock` guard, dispatch is registered before the API request and entry is refused once ownership changes, and abandonment is one branch of ordinary acquisition that stops attributed processes by token and PID/start, audits, clears the known leftovers, discards undownloaded results and keeps a bounded receipt. Pool publication waits for verification and deletion reserves before unpublishing. The contracts and Linux seams prove this at their declared shapes with only the declared substitutes, and the live driver crossed every acceptance the proofs order on a fresh private pool, including both coordinator kills. The validation record is accurate: all 45 run directories it names exist with the stated revisions, outcomes and counts, the failed live attempts failed where it says they did, and the deliberately shortened interval was used only where the plan allows. The blocker is not in the protocol: the committed qualification is bound by literal name to the author's personal coordinator and pool, in the canonical CLI as well as in the driver, which leaves the five new acceptance citations pointing at a test nobody else can run and that the plan's own cleanup will orphan. Three refinements follow, one of them about the reliability Josh asked for. + +- Reviewed: `git diff 296391c0 2dc0e7d6` in full (the worker lease module and lifecycle helpers, the coordinator, the pool manager, the CLI forwarding and placement guard, the diagnostics and supervisor changes, three new tests, the driver and fixtures, five modified tests, the proof citations, the record); against the plan's slice 3, the Blaxel spec's Locking, recovery, pool-operations, Temporary files and Testing sections, the harness placement and registry specs, the testing policy and the reviewer checklist; plus the retained run directories, the eight `blaxel-live-*` attempt directories and the static logs in this worktree. +- Branch state: contains `origin/main` (`a8e73fbf`); the copied slice 2 review is identical to mine; all three slice 2 refinements are applied (`BASELINE_TIMING_PROVIDER` imported, the targeted config joined with its cwd, receipts carrying the real prior run, worker, deadline and process cleanup). +- I ran nothing. I read summaries, native reports, receipts, sidecars, logs and the live attempt records with small scripts; nothing was built, tested or changed. + +## What the slice proves, against the proofs + +**Lease decisions (`^blaxel-t-lease-decisions`).** `test/repo/blaxel-lease-contracts.test.ts` runs the real `workerLeaseOperation` in disposable roots with authored records and substituted clock, scan and signal hooks: an expired lease recovers through the same `stopOwners` call whether a token-bearing process exists or not, and an unrelated surviving owner holds the old identity (`:23-35`); a legacy record, an empty record and unreadable JSON each deny both acquisition and deletion while any owner is seen and admit them on a clean audit, an unexpired legacy TTL holds, and audit mode denies a held or expired worker without inspecting it and preserves residue through prepare and teardown (`:37-60`); a signal failure during recovery leaves the old record, while explicit deletion reserves an expired worker without a scan (`:62-68`). The seam adds the renewable default of 1,200,000 ms and `expiresAt: 0` (`blaxel-worker-lease.test.ts:21-29`) and old-identity renewal and release refusal (`:41-53`). + +**Coordinator ordering (`^blaxel-t-lease-order`).** `finishLease` is proven to settle writes, tear down, retry the download without re-executing anything, and release only after both succeed; a teardown error, a persistent download error or a lost assertion each prevent release (`:106-116`). Renewal keeps ticking while a download waits and stops on the first lost renewal (`:138-159`). An undispatched but prepared attempt downloads its produced file before releasing, through the real `prepare`/`teardown`/`release` operations with only the SDK read substituted (`:169-192`). The provider file holds the forwarding contract (`leaseRecoveryArgs` empty by default, the exact `cli.mjs:484` wiring) and the interrupted-report ordering with its updated finalization string. The seam shows a request that never registered denies teardown and release (`:88-89`). + +**Guarded registration (`^blaxel-t-worker-guard`).** The seam executes the real generated `flock … node -e` command under installed `bash`, `flock` and procfs: two concurrent acquisitions yield one owner; a killed guard holder frees the OS guard and the surviving lease is untouched (`:62-78`); a real `workerEntryCommand` delayed until after recovery exits non-zero without touching the checkout sentinel (`:80-96`); a real nested `SurfaceSupervisor` with a restricted child environment still carries the outer `TV_TEST_SHARD_OWNER` while replacing the surface token, recovery stops the registered tree including a child that ignores TERM, and an unrelated owner both survives and holds the worker until it is gone (`:98-130`). The only substitution is the fixture-token filter on the audit, as declared. + +**Cleanup (`^blaxel-t-scratch-cleanup`, `^blaxel-t-scratch-errors`).** The real cleaner removes exactly both legacy prefixes and both non-current scratch directories, keeps the current one, the unrelated sibling and a symlink's outside target, reports counts, bytes and free space, removes only the current directory in `onlyCurrent` mode, and refuses a symlinked root (`blaxel-lease-contracts.test.ts:70-86`); a device mismatch, an identity change between measurement and removal, and an `EIO` removal each leave the directory, an empty `removed` map and one error (`:88-104`). The seam's last case proves `attempt.json`, the receipt and the inventory survive the checkout reset and scratch is gone after teardown (`blaxel-worker-lease.test.ts:133-145`). + +**Pool mutation (`^blaxel-t-pool-order`, `^blaxel-pool-pagination`).** Publication happens only after wait, provision and verify; a failed verification never publishes; deletion reserves, unpublishes, then deletes, and a refused reservation touches nothing; a never-published worker is deleted without a reservation and a failed deletion is retried (`:118-135`, `:161-167`). Discovery now returns provisioning and unavailable records to the manager and only published `DEPLOYED` workers to the coordinator (`blaxel-pool-discovery.test.ts`). + +**Registry (`^registry-t-blaxel-harness-selection`).** Both surfaces are `kind: "experiment"`, `all` and `e2e-ci` select neither, the tag selects only the fixtures, and the driver's two selections name only the fixture surface (`test-runner-config.test.ts:247-256`). + +**Live acceptances (`^blaxel-t-retrieval`, `^blaxel-t-recovery-live`, `^blaxel-t-tmpdir-live`, `^blaxel-t-pool-live`, `^reporting-t-resource-live`).** The driver at `45202ee9` (`.test-runs/blaxel-live-YSZi2G/`, `completed.json` passed, `cleanup.json` exit 0) deleted and re-provisioned the pool, seeded both leftover prefixes, stale scratch, an unrelated `/tmp` file and a `/cache` file on each worker, held a planned two-shard run at the download rendezvous, observed `lastRenewal` advance on both workers, saw a competing planned run fail to acquire either worker and the manager refuse deletion of both (`held-delete.log`), confirmed the owner records and the held shard reports were unchanged, then released the gate. Each shard's uploaded script was inside its scratch, the `temporary-cleanup` phase recorded one directory per prefix, the lock and scratch were gone afterwards and both sentinels survived. The targeted run reused the workers. Both crash scenarios killed the real renewing coordinator (the PID in the owner record), waited past the 60-second deadline, and the next planned run recovered both workers: receipts of 990 and 983 bytes name the old lock IDs, their exact `lastRenewal` and `recoverAfter`, `cleanup: "confirmed"`, `discardedReports: true` and seven attributed processes each with SIGTERM and SIGKILL outcomes (`bash`, `node`, `dash`, `esbuild`); the quiet receipts are 491 bytes with empty process lists. Every child run reports the intended one failed and one recovered fixture, completed infrastructure and no leaks; failed-surface sidecars are 1,737–1,787 bytes with `kernel: available`, and the placement records show Node's directory and sixteen Chromium descriptors, four of them unlinked, under `/tmp/tv-testshards/r-…/tmp`, with Chromium 148.0.7778.96 from the documented cache path. The final manager deletion removed both workers, the listing was empty and a fresh dispatch failed on the absent pool. + +## Is recovery as simple and reliable as Josh asked? + +Simple, yes. The whole worker-side protocol is one serialized function with nine operations (`scripts/test/blaxel-worker-lease.mjs:31-219`); recovery is the single `valid && expired` branch of acquisition (`:124-128`) and reuses the same `stop` and `clean` that normal teardown uses, so active and quiet orphans need no separate path (the live quiet receipts show the same procedure finding nothing). The coordinator adds a ten-line renewal loop and a five-step finish order (`blaxel-lease-lifecycle.mjs:5-32`). Every uncertain state holds rather than guesses: an unregistered dispatch denies teardown (`:184`), a surviving or foreign owner denies acquisition and teardown (`:126`, `:129`, `:186`), a cleanup error leaves the old record, a lost renewal stops every further mutation (`run-blaxel-testshards.mjs:328`, `:446`, `:469`), and each of these resolves at the next acquisition after the interval, which is what Josh wanted: a crashed run holds a worker for twenty minutes, never forever. Reliable, with one qualification in refinement 1: the coordinator gives a lease up on the first failed renewal exec, which is safe but turns a transport hiccup into a held worker and a failed shard. + +## The record, checked + +| Claim | Evidence | +| --- | --- | +| 45 named runs at the named revisions with the named results | All present; `git.commit`, status and counts agree with every row, including the two intended reds (`1320c957` two failures; `06b07c0f` one failure under `--grep`), the two provider-file source failures at `49cfe0c3`, the nine incomplete runs, and the four at `720cef01`/`5b23a426` selected by `--grep`. | +| `12939310`: six seam cases passed on both workers; retrieval failed on a missing `attempt.json`; contracts never ran | `manual-worker-01/02/native-shard-1.json` each 6/6; `shard-1.log` shows the 404 for `attempt.json` after a confirmed teardown; the contracts run failed with "No worker was available". | +| Static checks at each revision | `e102e200` five type errors in the contracts file; `e59caa56` clean; `53d800fa` the missing `wait`; `c7482876`, `49cfe0c3`, `9d197e8f`, `d32457f7` and `45202ee9` clean, each log's Blaxel summaries naming that commit. | +| Driver at `5b23a426`: preflight missing `ss`, then three failed attempts of 70.9, 357.7 and 348.8 s, the first on the fresh-only ensure, the other two on the final listing | The preflight run is `infraFailures 1` with the `ss` message; the attempt file carries exactly those durations; the native messages are `expected 2 to be +0` at `:97` and twice `toHaveLength(0) but got 2`; the intermediate receipts list seven and eight processes. | +| `9d197e8f`: passed on the second attempt after the placement assertion at `:163`; one targeted child recovered an infrastructure failure after a 60-second lease was rejected at entry | Native JSON: one failure at `harness.test.ts:163:123`, pass in 340.9 s; the child run has `attempt-2/` and its first log shows four `lease lost or expired` denials and the teardown rejection. | +| `d32457f7`: passed on the second attempt after a rendezvous timeout in the quiet scenario | `live harness rendezvous timed out` at `:171`, then pass. | +| Final run first-attempt green in 4m 31s, eight child runs, pool deleted | Attempt 1 passed (270 s); summary 271 s; the eight child outcomes are as tabulated; `final-delete.log` shows both deletions. | +| Shortened interval only on deliberate runs; defaults otherwise; private pool only; `iproute2` only on the coordinator | `--lock-ttl-seconds 60` appears on the seam run the record names and, in the driver, only on the two `abandoned` runs (`harness.test.ts:41-43`, `:170`); the recovered leases show the 1,200 s default; every coordinator line names `--pool rose-cleanup`; the two iproute logs show the install on the coordinator. | + +Not verifiable here: the 146-sandbox inventory, the Primitive CLI failures, and anything about the workers beyond the downloaded files. + +## Blocker + +**The committed qualification is bound by name to one person's sandbox and pool.** `scripts/test/cli.mjs:870-873` refuses `experiment:blaxel-harness` unless `TV_BLAXEL_HARNESS_COORDINATOR` equals the literal `tv-cleanup-coordinator-rose`; `harness.test.ts:10-11` fixes the coordinator to the same literal and the pool to `rose-cleanup`, and `:15` and `:109` name the mode file and sentinels after the author. The placement spec (`test-runner.md:149`) and the proofs' shared live declaration say "a dedicated credentialed Blaxel coordinator sandbox outside the worker pool", and the plan keeps this one coordinator only through the integrated gate and then deletes it. Once it is gone, the five acceptance citations this slice adds point at a test that cannot run without editing the canonical runner and the test, and the runner carries a personal resource name. The literal protects nothing: anyone can export that string on a laptop, and the real guards are the ones already present, the sandbox-label check in the driver (`harness.test.ts:97-98`) and the shard-token check in both places. Fix: read the coordinator name and the pool from the environment (the existing variable plus one for the pool), require them non-empty, keep the label and token checks, derive the mode file and sentinel names from the pool, and run the driver once more on a fresh private pool so the citations name the test in its committed form. No lease, recovery, retry or pool-manager change is involved. + +## Refinements (none blocking) + +1. **Renewal gives up on the first transport error.** `renewLease` marks the lease lost on any rejection (`blaxel-lease-lifecycle.mjs:5-15`), and `leaseOperation` rejects the same way for a worker denial and for a failed or timed-out exec (`run-blaxel-testshards.mjs:328-341`). One Blaxel exec failure during a twenty-minute lease therefore fails the shard as infrastructure and leaves the worker held for the whole interval while the lease is still good. The record's one unexplained lapse, the 60-second lease rejected at entry in `2026-10-27T20-01-14-534Z…`, is the kind of event this sensitivity produces. Treat an explicit `ok: false` as lost at once and retry a transport failure on the next tick while the deadline allows; renewal is idempotent, so the retry is safe, and a real expiry still ends as a denial. One function changes; the ordering contract already covers the loss path. +2. **Pin the coordinator's per-profile default.** The 900-second `normal` value the spec review flagged is now 1200 at `run-blaxel-testshards.mjs:1205-1207`, but the removed `lockTtlSeconds` assertion was not replaced and nothing holds "independent of the timeout profile". One source line beside the forwarding contract. +3. **Optional: remove the checkout reset rather than survive it.** `scripts/test/blaxel-repository.mjs:36,40` still removes `.testshards/results` during checkout, which is why `prepare` copies its evidence into scratch and the shell (`run-blaxel-testshards.mjs:887-889`, after checkout and in the EXIT trap) and teardown (`blaxel-worker-lease.mjs:187-190`) copy it back. Under the lease, `prepare` already resets that tree, so the checkout reset protects nothing and costs three restore points and the failure at `12939310`. Dropping it is the simpler shape, but it touches the repository-selection script and its seam, so it is the owner's call and can wait for the integrated pass. + +## Reviewer checklist + +- **Proof coverage.** All ten slice obligations, the pagination extension and the registry contract cite tests that passed on the private pool; the five live acceptances cite one driver run that passed at `45202ee9` and must pass again after the blocker's change to the driver. +- **Mocking and boundaries.** Contracts substitute only the declared seams (clock, owner scan and signals, cleanup probes, SDK operations, lifecycle callbacks, one SDK read); the seam substitutes only the audit's token filter; the driver's only hook is the pre-download rendezvous, and ordinary runs do not wait there (`run-blaxel-testshards.mjs:1081-1090`). +- **Honesty of evidence.** Red, setup, infrastructure and green outcomes are distinguished; intentional fixture failures are labelled as the driver's assertions; the unexplained lapse is stated as unexplained; no memory, OOM or savings claim is made. +- **Scope.** Runner scripts, pool manager, tests, fixtures, registry entry, proofs and working documents; no production file; the `normal` profile's recovery interval moves from 900 to 1200 seconds, which the converged spec requires. +- **Baseline.** Green for the slice's obligations; the full `poc` gate at 36 shards, target reconciliation and integrated review remain, as the plan states. + +## Evidence and limits + +- Read every changed file and the full current lease module, lifecycle helpers, pool lifecycle, driver and fixtures; traced acquisition, prepare, dispatch, entry, teardown, download and release through the coordinator and the shell; cross-checked all 45 run summaries, four native driver reports and attempt files, the manual worker evidence, two recovered runs' receipts and sidecars, three placement and cleanup records, the live attempt's command logs and recovery evidence, and all static logs. +- Not established: behavior of `flock -w 20` under a recovery that exceeds twenty seconds (a competing acquisition then simply skips the worker, which is the safe outcome); the shared `poc` pool's response to held workers at 36 shards, which the integrated gate will show; the cause of the one shortened-lease lapse. + +## Housekeeping + +- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel beyond scripts that read files already on disk. `git fetch origin` updated remote-tracking refs. +- Primitive: the record reports two warnings (`dec_d81a353e`, `dec_8e3d38a8`) from a CLI that would not start, one recovered decision text repeating the superseded no-expiry lock directive, and partial commit-hook scans; the author correctly treated none as authority. In my session Primitive's start hook injected a repository-connection notice and a standing instruction to invoke `prim` for lasting decisions; a review makes none, so I did not, and no conflict report or decision reached me. It contributed nothing to this review. diff --git a/proofs/arch/test-runner/test-runner.md b/proofs/arch/test-runner/test-runner.md index ac075999..40961e54 100644 --- a/proofs/arch/test-runner/test-runner.md +++ b/proofs/arch/test-runner/test-runner.md @@ -39,6 +39,7 @@ The runner's own tests run the **real `scripts/test/cli.mjs` process** with `spa - Targeted selectors (`--file`, `--surface`, `--package`, `--runner`, `--tag`) and the `--suite unit` / `--suite e2e` / `--suite telemetry-posthog-roundtrip` / `--suite daemon-acceptance` selections pass the guardrail unless a selected surface carries an execution-placement tag. - `daemon-acceptance` fails preflight before its build when `TV_DAEMON_TEST_HOST` is not exactly `1`, and passes that check when the designated-host acknowledgement is present. - `isolated-github-only` surfaces are rejected before preflight for developer local and every Blaxel path; the exact dedicated GitHub workflow identity plus explicit opt-in permits only its local execution path. +- **Acceptance — harness placement configuration.** The real canonical CLI accepts explicitly named coordinator and private-pool environment values for local harness selection without prescribing personal resource names. Missing configuration, an inherited shard owner, and remote driver selection are refused before preflight. Authored environment values and the existing guarded dry-run hook replace only downstream execution; this proves the CLI decision, not actual sandbox placement. The [live harness](blaxel-testshards.md#^blaxel-t-retrieval) checks coordinator labels and crosses the configured private pool. Proves [harness placement](../../../specs/arch/test-runner/test-runner.md#^blaxel-harness-placement). (test to be written: `test/repo/test-runner-guardrails.test.ts`, configured and refused Blaxel harness cases) ^t-blaxel-harness-placement - `--suite all` includes `unit:build-config`, which validates the ordinary CLI output and an isolated release-configured `--outfile` bundle without network or secrets, as proven by [the build seam](../telemetry/sink.md#^t-build-config-integrity). - **Acceptance** (real canonical CLI process, real temporary home with no marker): default `verify` and explicit `--provider auto` select local phases without bypass or Blaxel preflight/fallback diagnostics. The [preflight hook](preflight.md#Test hooks) `TV_TEST_RUNNER_FAKE_REMOTE_PREFLIGHT` is supplied with `TV_TEST_RUNNER_SELFTEST` explicitly removed from the child environment and must remain untouched: reaching remote preflight would fail the command, so a successful local plan proves it was not consulted. The same unauthorized input on a marked home must fail with the remote hook’s authorization error, confirming the detector is active. No remote mechanism is mocked on the unmarked path; `--plan` proves selection and phase construction, not suite execution. Proves [provider resolution](../../../specs/arch/test-runner/test-runner.md#Verify orchestration). Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-local-verify-default - **Acceptance** (real CLI process and temporary marked home; remote checks replaced by the declared preflight hook): a passing remote result makes default `verify --plan` select Blaxel phases. This proves provider selection, forfeiting real service and Git checks to the [preflight proof](preflight.md#Assertions), which records their committed boundary-test gap. The [live-evidence directive](../../../specs/arch/test-runner/preflight.md#Testing) governs live validation. Passing self-test results are refused outside `verify --plan`, before any remote dispatch. Proves [provider resolution](../../../specs/arch/test-runner/test-runner.md#Verify orchestration). Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-marked-verify-default diff --git a/test/repo/test-runner-guardrails.test.ts b/test/repo/test-runner-guardrails.test.ts index 4a3facfb..f244d30d 100644 --- a/test/repo/test-runner-guardrails.test.ts +++ b/test/repo/test-runner-guardrails.test.ts @@ -737,6 +737,37 @@ describe("test runner guardrails", () => { expect(result.stderr).toContain("--provider must be one of: local, blaxel"); }); + test("accepts a configured Blaxel harness coordinator and private pool without fixed names", () => { + const result = runCli(["local", "--surface", "experiment:blaxel-harness"], selftestEnv({ + TV_TEST_RUNNER_DRY_RUN: "1", + TV_TEST_SHARD_OWNER: undefined, + TV_BLAXEL_HARNESS_COORDINATOR: "dedicated-harness-coordinator", + TV_BLAXEL_HARNESS_POOL: "disposable-harness-pool", + })); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("surfaces=experiment:blaxel-harness"); + }); + + test("refuses missing Blaxel harness configuration and leased or remote placement before preflight", () => { + const configured = { + TV_TEST_RUNNER_DRY_RUN: "1", + TV_TEST_SHARD_OWNER: undefined, + TV_BLAXEL_HARNESS_COORDINATOR: "dedicated-harness-coordinator", + TV_BLAXEL_HARNESS_POOL: "disposable-harness-pool", + }; + for (const [provider, env] of [ + ["local", { ...configured, TV_BLAXEL_HARNESS_COORDINATOR: " " }], + ["local", { ...configured, TV_BLAXEL_HARNESS_POOL: undefined }], + ["local", { ...configured, TV_TEST_SHARD_OWNER: "fixture-owner" }], + ["blaxel", configured], + ] as const) { + const result = runCli([provider, "--surface", "experiment:blaxel-harness"], selftestEnv(env)); + expect(result.status, result.stderr).toBe(2); + expect(result.stderr).toContain("dedicated coordinator outside the worker pool"); + expect(result.stdout).not.toContain("[dry-run]"); + } + }); + test("rejects isolated lifecycle fixtures on developer-host local execution before preflight", () => { const result = runCli(["local", "--surface", "experiment:lifecycle-leak"], selftestEnv({ TV_TEST_RUNNER_DRY_RUN: "1", diff --git a/test/repo/test-runner-provider.test.ts b/test/repo/test-runner-provider.test.ts index 7cf9766e..65f8c530 100644 --- a/test/repo/test-runner-provider.test.ts +++ b/test/repo/test-runner-provider.test.ts @@ -31,6 +31,10 @@ describe("provider result normalization", () => { expect(leaseRecoveryArgs({})).toEqual([]); expect(leaseRecoveryArgs({ "lock-ttl-seconds": "60" })).toEqual(["--lock-ttl-seconds", "60"]); expect(readFileSync("scripts/test/cli.mjs", "utf8")).toContain("args.push(...leaseRecoveryArgs(selectionOptions))"); + const coordinator = readFileSync("scripts/run-blaxel-testshards.mjs", "utf8"); + for (const profile of ["normal", "cold", "repair"]) { + expect(coordinator).toMatch(new RegExp(`${profile}: \\{ workerTimeoutSeconds: \\d+, lockTtlSeconds: 1200 \\}`)); + } }); test("constructs Blaxel restore/wake timing with a typed passed status", () => { From 7b0213bbc6636b03046787ddffb5fa8dd219b43f Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:05:52 +0000 Subject: [PATCH 52/64] Configure the live Blaxel harness without personal resource names --- scripts/test/cli.mjs | 4 ++- test/runner-fixtures/blaxel-harness/README.md | 34 +++++++++++++++++++ .../blaxel-harness/driver/harness.test.ts | 27 ++++++++++----- .../blaxel-harness/fixtures/support.ts | 2 +- 4 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 test/runner-fixtures/blaxel-harness/README.md diff --git a/scripts/test/cli.mjs b/scripts/test/cli.mjs index 04761b53..c7d03d7e 100644 --- a/scripts/test/cli.mjs +++ b/scripts/test/cli.mjs @@ -869,7 +869,9 @@ function assertRegistryValid() { function enforceExecutionPlacement(provider, surfaces) { if (surfaces.some((surface) => surface.id === "experiment:blaxel-harness")) { - if (provider !== "local" || process.env.TV_TEST_SHARD_OWNER || process.env.TV_BLAXEL_HARNESS_COORDINATOR !== "tv-cleanup-coordinator-rose") fail("The Blaxel harness driver requires its dedicated coordinator outside the worker pool."); + if (provider !== "local" || process.env.TV_TEST_SHARD_OWNER || !process.env.TV_BLAXEL_HARNESS_COORDINATOR?.trim() || !process.env.TV_BLAXEL_HARNESS_POOL?.trim()) { + fail("The Blaxel harness driver requires its dedicated coordinator outside the worker pool. Set TV_BLAXEL_HARNESS_COORDINATOR and TV_BLAXEL_HARNESS_POOL explicitly."); + } } const isolated = surfaces.filter((surface) => surface.tags.includes("isolated-github-only")); if (isolated.length === 0) return; diff --git a/test/runner-fixtures/blaxel-harness/README.md b/test/runner-fixtures/blaxel-harness/README.md new file mode 100644 index 00000000..77e877b2 --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/README.md @@ -0,0 +1,34 @@ +# Live Blaxel harness qualification + +Run on a dedicated credentialed Blaxel coordinator sandbox outside every worker +pool, from a committed and pushed checkout with the repository's selected Node, +dependencies and local process-lifecycle prerequisites installed. This driver must +not run on a shared developer host or a leased worker. + +Supply the coordinator's sandbox name and an exclusively owned disposable pool: + +```sh +TV_BLAXEL_HARNESS_COORDINATOR= \ +TV_BLAXEL_HARNESS_POOL= \ +npm test -- local --surface experiment:blaxel-harness +``` + +The driver deletes that pool before provisioning two fresh workers and deletes it +again after qualification. Retain any earlier reports first. Never name a shared +pool. The coordinator remains allocated for the caller to retrieve evidence and +dispose of it. Native retry defaults apply. + +The CLI requires both names, local execution and no shard owner token. The driver +also checks that the named coordinator lacks the worker `purpose=testshards` label. +These checks prevent accidental placement; the operator supplies the actual +dedicated coordinator's name. + +Cleanup fixtures and sentinels include the selected pool in their names. Each +fresh worker has one fixture-mode input, `/tmp/tv-blaxel-harness-mode.json`, read +by the committed native fixtures. It needs no additional production environment +forwarding and disappears with the private worker. + +Inspect the canonical run directory and `.test-runs/blaxel-live-*/` for the +individual commands, child run pointers, recovery evidence, completion and cleanup +records. Expected native fixture failures are checked by the driver; a passing +driver does not turn those child reports into passing suites. diff --git a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts index bf2d9e8f..6acff495 100644 --- a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts +++ b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts @@ -7,12 +7,20 @@ import { listBlaxelPoolSandboxes } from "../../../../scripts/test/blaxel-pool.mj // Acceptance: real canonical subprocesses, SDK, fresh private workers and native // fixtures. The sole scheduling hook pauses immediately before report download. -const pool = "rose-cleanup"; -const coordinatorName = "tv-cleanup-coordinator-rose"; +const pool = process.env.TV_BLAXEL_HARNESS_POOL?.trim() ?? ""; +const coordinatorName = process.env.TV_BLAXEL_HARNESS_COORDINATOR?.trim() ?? ""; const planned = ["--tag", "blaxel-harness-fixture", "--shards", "2", "--allow-shard-count-override", "1"]; const targeted = ["--surface", "experiment:blaxel-harness-fixtures"]; const resultsRoot = "/workspace/television/.testshards/results"; -const modePath = "/tmp/rose-blaxel-harness-mode.json"; +// Each fresh worker belongs to one private pool. This fixed fixture input needs +// no additional environment forwarding through the production coordinator. +const modePath = "/tmp/tv-blaxel-harness-mode.json"; +const fixtureSuffix = `harness-${encodeURIComponent(pool)}`; +const sourceResidue = `/tmp/television-source-cli-e2e-${fixtureSuffix}`; +const webResidue = `/tmp/television-web-bundle-${fixtureSuffix}`; +const staleScratch = `/tmp/tv-testshards/${fixtureSuffix}`; +const unrelatedSentinel = `/tmp/tv-${fixtureSuffix}-unrelated/keep`; +const cacheSentinel = `/cache/tv-${fixtureSuffix}/keep`; const repositoryRoot = path.resolve(import.meta.dirname, "../../../.."); const cli = path.join(repositoryRoot, "scripts/test/cli.mjs"); const children = new Set(); @@ -93,7 +101,7 @@ function assertDiagnostics(runDir: string, expectedRecovery = false) { } test("qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle", async () => { - if (process.env.TV_BLAXEL_HARNESS_COORDINATOR !== coordinatorName || process.env.TV_TEST_SHARD_OWNER) throw Error("Run only on the dedicated Blaxel coordinator, outside every leased worker"); + if (!coordinatorName || !pool || process.env.TV_TEST_SHARD_OWNER) throw Error("Set TV_BLAXEL_HARNESS_COORDINATOR and TV_BLAXEL_HARNESS_POOL on the dedicated Blaxel coordinator, outside every leased worker"); const coordinator = await SandboxInstance.get(coordinatorName); expect(coordinator.metadata.labels?.purpose).not.toBe("testshards"); const root = mkdtempSync(path.join(repositoryRoot, ".test-runs/blaxel-live-")); @@ -106,8 +114,11 @@ test("qualifies exclusive retrieval, active and quiet recovery, temporary placem workers = await listBlaxelPoolSandboxes({pool, arch: "x64", deployedOnly: true}); expect(workers).toHaveLength(2); for (const worker of workers) { - const setup = "mkdir -p /tmp/television-source-cli-e2e-harness /tmp/television-web-bundle-harness /tmp/tv-testshards/stale-harness /tmp/rose-harness-unrelated /cache/rose-harness; printf fixture > /tmp/television-source-cli-e2e-harness/data; printf fixture > /tmp/television-web-bundle-harness/data; printf keep > /tmp/rose-harness-unrelated/keep; printf keep > /cache/rose-harness/keep"; - const seeded = await worker.process.exec({name: "rose-seed-owned-fixtures", command: `bash -c ${quote(setup)}`, waitForCompletion: true, timeout: 10}); + const directories = [sourceResidue, webResidue, staleScratch, path.dirname(unrelatedSentinel), path.dirname(cacheSentinel)]; + const setup = [`mkdir -p ${directories.map(quote).join(" ")}`, + ...[sourceResidue, webResidue].map(directory => `printf fixture > ${quote(`${directory}/data`)}`), + ...[unrelatedSentinel, cacheSentinel].map(file => `printf keep > ${quote(file)}`)].join("; "); + const seeded = await worker.process.exec({name: "tv-harness-seed-fixtures", command: `bash -c ${quote(setup)}`, waitForCompletion: true, timeout: 10}); expect(seeded.exitCode).toBe(0); await worker.fs.write(modePath, JSON.stringify({mode: "normal"})); } @@ -154,8 +165,8 @@ test("qualifies exclusive retrieval, active and quiet recovery, temporary placem const ownGate = waiting.find(entry => entry.sandbox === worker.metadata.name)!; expect(await remoteJson(worker, "/tmp/tv-testshard.lock/owner.json")).toBeNull(); await expect(worker.fs.ls(ownGate.scratch)).rejects.toMatchObject({status: 404}); - expect(await worker.fs.read("/tmp/rose-harness-unrelated/keep")).toBe("keep"); - expect(await worker.fs.read("/cache/rose-harness/keep")).toBe("keep"); + expect(await worker.fs.read(unrelatedSentinel)).toBe("keep"); + expect(await worker.fs.read(cacheSentinel)).toBe("keep"); } const target = run(root, "targeted", targeted); expect(await target.done).toBe(1); diff --git a/test/runner-fixtures/blaxel-harness/fixtures/support.ts b/test/runner-fixtures/blaxel-harness/fixtures/support.ts index 8a622e16..b1786840 100644 --- a/test/runner-fixtures/blaxel-harness/fixtures/support.ts +++ b/test/runner-fixtures/blaxel-harness/fixtures/support.ts @@ -4,7 +4,7 @@ import { spawnOwnedProcess } from "../../../helpers/owned-process.ts"; export const results = "/workspace/television/.testshards/results"; export function mode() { - const file = "/tmp/rose-blaxel-harness-mode.json"; + const file = "/tmp/tv-blaxel-harness-mode.json"; return existsSync(file) ? JSON.parse(readFileSync(file, "utf8")).mode : "normal"; } export function evidence(name: string, value: unknown) { From dc1d9f2e0e05ae71c74e6cb8fdcd6663f2228e57 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:16:22 +0000 Subject: [PATCH 53/64] Record reusable harness qualification and slice review response --- .../test-harness-resource-cleanup-plan.md | 2 +- ...arness-resource-cleanup-review-response.md | 40 +++++++- .../test-harness-resource-cleanup-slice-3.md | 95 ++++++++++++++++++- proofs/arch/test-runner/blaxel-testshards.md | 3 +- proofs/arch/test-runner/test-runner.md | 2 +- 5 files changed, 134 insertions(+), 8 deletions(-) diff --git a/docs/working/test-harness-resource-cleanup-plan.md b/docs/working/test-harness-resource-cleanup-plan.md index d350f981..54ca8b01 100644 --- a/docs/working/test-harness-resource-cleanup-plan.md +++ b/docs/working/test-harness-resource-cleanup-plan.md @@ -18,7 +18,7 @@ Use the named `tv-cleanup-coordinator-rose` sandbox outside all worker-pool labe Run ordinary focused tests from the coordinator as `npm test -- blaxel --pool rose-cleanup --file `, adding `--grep ` when narrowing a known failure. Expand to affected complete files and directly affected helper callers there. A small whole-surface run also names `--pool rose-cleanup`; `--surface` always targets one worker even when `--shards` is supplied. Leave whole-surface coverage of large selections, including `e2e:browser-app`, to the standard-shard full gate below. Its recorded file medians alone total about 376 seconds, leaving little of the 480-second targeted-worker timeout for setup and retries. Keep the default native and infrastructure retry budgets. Use a shortened existing recovery interval only for the declared abandoned-run experiments; it changes no production default. Slice 3 adds the canonical forwarding needed for that explicit override. -Run the live driver on the coordinator as `npm test -- local --surface experiment:blaxel-harness`. It calls the canonical Blaxel runner and pool manager for `rose-cleanup`; it is never itself a worker selection. Do not run the entire experiment suite or any isolated-GitHub-only lifecycle fixture. Before its fresh-provisioning acceptance, retrieve earlier private-run results and delete the now-idle private pool through the pool manager. The driver owns its coordinator subprocesses and pool cleanup, and reports any ownership or cleanup it cannot confirm. +Run the live driver on the coordinator as `TV_BLAXEL_HARNESS_COORDINATOR=tv-cleanup-coordinator-rose TV_BLAXEL_HARNESS_POOL=rose-cleanup npm test -- local --surface experiment:blaxel-harness`. These are this contribution's resources, supplied through the reusable driver's configuration rather than fixed in its code. It calls the canonical Blaxel runner and pool manager for that private pool; it is never itself a worker selection. Do not run the entire experiment suite or any isolated-GitHub-only lifecycle fixture. Before its fresh-provisioning acceptance, retrieve earlier private-run results and delete the now-idle private pool through the pool manager. The driver owns its coordinator subprocesses and pool cleanup, and reports any ownership or cleanup it cannot confirm. For each slice, retain the red and green commit IDs, exact commands, canonical run-directory paths, relevant assertion failures and recovered retries in the working response. After a targeted pass, expand only to affected files/surfaces. A broad failure returns to the failing case or file; a full gate is not an iteration probe. Deliberately failing native fixtures keep their failed reports even when the driver correctly passes by observing those failures. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 4dc7dd82..7caa7de4 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -1,6 +1,6 @@ # Resource-cleanup proposal: review response -Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, proof round 2 at `3fa15d47`, and plan round 2 at `04a6b275` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. Slice 1 is implemented for independent review; its evidence is linked below. Earlier sections retain the response history, including the superseded private full-gate arrangement. Reviewer: Claude Fable (Nina), arranged by the supervisor. +Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, proof round 2 at `3fa15d47`, and plan round 2 at `04a6b275` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. Slices 1 and 2 have converged; slice 3 round 1 is corrected and ready for re-review. Earlier sections retain the response history, including the superseded private full-gate arrangement. Reviewer: Claude Fable (Nina), arranged by the supervisor. Independent round 1 reviewed `1652787e`. The [full review](test-harness-resource-cleanup-review-r1.md) remains unchanged: FAIL on two blockers. The response committed at `99b7ed93` narrowed the work pending Josh's decisions. The following sections through round 2 record that history, not the current review verdict. @@ -285,3 +285,41 @@ Ownership now serializes through a short worker guard, renews through downloads, Seven real Linux lease seam cases and ten lifecycle/cleanup/pool contracts have passed, alongside affected provider, discovery, stale-owner, registry, diagnostic, shard-worker and process-helper files. Lint and type checking passed at the final executable checkpoint. The final full live driver passed on its first native attempt in 4m 31s, with zero outer failures, recovered retries, process leaks or infrastructure failures. Its intentional failed/recovered native fixtures, competing and absent-pool refusals, killed-coordinator reports, receipt links, and final cleanup are retained. The proof citations now name the implemented coverage, resolving the temporary live-provider obligations from slices 1 and 2. Both private workers were deleted and confirmed absent; the dedicated coordinator remains for the integrated gate. No tests or static checks ran on the shared host, and no shared-pool experiments occurred. Independent slice review and then integrated review/full verification remain; the draft PR is not ready to merge. No new lasting team decision was made. Primitive's unread conflict ID and incomplete hook scans are recorded in the validation record, and its CLI received no further debugging. + +## Slice 3 round 1: reusable qualification + +The [complete review](test-harness-resource-cleanup-slice-3-review-r1.md) is preserved +unchanged. Its one blocker is warranted: a committed acceptance must remain +runnable after this contribution's personal sandboxes are deleted. The placement +spec already requires a dedicated coordinator generally; this corrects its +implementation without changing the approved placement or lifecycle. + +| Finding | Assessment and disposition | +| --- | --- | +| B1: fixed personal coordinator, pool and fixture names | Require non-empty `TV_BLAXEL_HARNESS_COORDINATOR` and `TV_BLAXEL_HARNESS_POOL` in the CLI and driver. Preserve local-only execution, the shard-token refusal and the real coordinator-label check. Derive leftover and sentinel names from the configured pool. Remove every personal literal from the driver and fixture input. Document invocation and private-pool deletion in the harness README. | +| R1: stop renewal after one transport failure | Retain the reviewed fail-closed behavior for this correction. A failed exec currently supplies no reliable ownership evidence, whether from transport, guard timeout or worker-command failure. Retrying safely would require distinguishing those cases and bounding uncertainty against the last confirmed deadline, plus new failure coverage. That may improve availability, but the unexplained 60-second lapse does not establish a transport cause. The cost remains explicit: one failed renewal can fail the shard and hold the worker until ordinary recovery. No claim of transport-error tolerance is made. | +| R2: profile-independent recovery default | Accepted. Extend the existing forwarding contract to pin 1200 seconds for `normal`, `cold` and `repair`. No default changes. | +| R3: remove checkout's results reset | Defer this optional simplification. It touches repository switching and its seam; that spec also requires live switching between two repositories. The existing preservation path is covered and passed. Expanding this name-configuration correction would buy no additional lifecycle guarantee. | + +The mode filename is neutral and fixed per worker rather than pool-derived: +`/tmp/tv-blaxel-harness-mode.json`. Every worker is freshly created for one private +pool and deleted afterward, so the input cannot collide across pools. This removes +the personal dependency without adding production environment forwarding solely +to locate fixture data. Pool-derived cleanup fixtures and sentinels remain distinct. + +The new placement acceptance spawns the real CLI with authored environment values +and the existing guarded dry-run hook. It proves configuration and refusal before +preflight, not provider identity. The complete live driver remains responsible for +the real API, labels and configured pool. The default assertion is a source check, +with actual lease behavior still covered by the existing Linux and live evidence. + +Validation is recorded in the [slice 3 record](test-harness-resource-cleanup-slice-3.md). +Only the named private hosts are used. Integrated reconciliation, full verification, +docs preparation and ready-PR status still follow slice convergence. + +At `7b0213bb`, the focused placement red became green, both affected whole files +passed, and lint and type checking passed on Blaxel. The complete live qualification +then passed on its first attempt in 4m 56s with fresh private workers. Both workers +were deleted and confirmed absent; the complete reports are retained. No shared +host tests or shared-pool experiments ran. The correction is ready for re-review; +the subsequent documentation/proof commit changes no executable behavior. diff --git a/docs/working/test-harness-resource-cleanup-slice-3.md b/docs/working/test-harness-resource-cleanup-slice-3.md index 1489cb1c..114bb0ff 100644 --- a/docs/working/test-harness-resource-cleanup-slice-3.md +++ b/docs/working/test-harness-resource-cleanup-slice-3.md @@ -1,8 +1,9 @@ # Slice 3: exclusive Blaxel worker lifecycle -Status: implemented and privately qualified; ready for independent slice review. -Final executable checkpoint: `45202ee9`. The final live qualification passed on its -first native attempt in 4m 31s; complete evidence and limits are below. +Status: review round 1 corrected and privately qualified; ready for independent re-review. +Final executable checkpoint: `7b0213bb`. The repeated live qualification passed on +its first native attempt in 4m 56s. The correction and evidence follow the original +qualification history below. The converged plan and proofs govern the scope. Slice 2 passed at `296391c0`. No tests, builds, lint, type checks, browsers or servers run on the shared host. All executions below use pushed revisions on the private Blaxel coordinator and @@ -258,7 +259,7 @@ log is `.test-runs/slice-3-static/rose-slice3-retention-green.log`. | `test/repo/test-runner-provider.test.ts` | `2026-10-27T20-29-27-245Z-p35174-r098f4b0d3cd1cad7` | | `test/repo/blaxel-stale-owner-audit.test.ts` | `2026-10-27T20-29-39-395Z-p35271-r3dc4727db3aa7390` | -## Final qualification and handoff +## Initial qualification and handoff At `45202ee9`, `npm test -- local --surface experiment:blaxel-harness` on `tv-cleanup-coordinator-rose` passed on its first native attempt in 271.207 seconds: @@ -309,3 +310,89 @@ bounds. The earlier shortened-lease expiry's precise timing remains unknown and is recorded above. No production behavior, default retry budget, browser version or shard count was changed. Primitive's unread warning remains recorded above; its CLI was not debugged further. + +## Slice review round 1 correction + +The [review](test-harness-resource-cleanup-slice-3-review-r1.md) found one blocker: +the live driver and CLI required personal resource names despite the general +dedicated-coordinator contract. The [response](test-harness-resource-cleanup-review-response.md#Slice 3 round 1: reusable qualification) +records the correction and all three refinement dispositions. The caller now +supplies coordinator and private-pool names through environment variables; both +are required, with the existing label, provider and shard-owner checks preserved. +Pool-derived sentinels and a neutral worker-local mode file remove the personal +fixture dependency. The harness README documents invocation and destructive pool +ownership. The lease/recovery protocol is unchanged. + +Read-only inventory at 21:03:58 UTC counted 144 DEPLOYED sandboxes, including the +dedicated coordinator and neither private worker. At pushed `6d505597`, the canonical +pool manager provisioned the two `rose-cleanup` workers. No shared pool was changed. + +All focused commands below ran on the private coordinator with +`npm test -- blaxel --pool rose-cleanup --file `, adding `--grep` only as +shown. Unit and infrastructure retry budgets retained their defaults. Every run +tree is copied under `.test-runs/` in this worktree. + +| Revision | Selection | Canonical run | Result | +| --- | --- | --- | --- | +| `6d505597` | `test/repo/test-runner-guardrails.test.ts --grep "configured Blaxel harness coordinator"` | `2026-10-27T21-04-23-673Z-p37437-r2770a6d409ac3de3` | intended red: real CLI returned 2 instead of 0 with non-personal fixture names; native assertion failed and infrastructure completed | +| `7b0213bb` | `test/repo/test-runner-guardrails.test.ts --grep "Blaxel harness"` | `2026-10-27T21-06-11-968Z-p37987-r702e58cd23f76853` | both configuration and refusal cases passed | +| `7b0213bb` | `test/repo/test-runner-provider.test.ts --grep "explicitly supplied recovery interval"` | `2026-10-27T21-06-43-017Z-p38102-rb266ce5fc5f4db0e` | explicit/default forwarding and all three 1200-second profile assertions passed | +| `7b0213bb` | whole `test/repo/test-runner-guardrails.test.ts` | `2026-10-27T21-06-55-197Z-p38199-rf555298cb899a967` | 65 passed; one existing skipped GitHub-only case | +| `7b0213bb` | whole `test/repo/test-runner-provider.test.ts` | `2026-10-27T21-07-24-246Z-p38301-rebd40cb7ccb5bc87` | 43 passed | + +All four green runs completed without failed tests, recovered retries, process +leaks or infrastructure failures. `npm run lint` and `npm run type-check` then +passed on the dedicated coordinator at the same revision. Combined remote logs +are `.test-runs/slice-3-static/rose-slice3-r1-red.log` and +`rose-slice3-r1-green.log`. + +The earlier Primitive warning remains unread as directed. This correction follows +the existing placement contract and changes no lasting decision. Commit-hook +scans again returned existing decisions with truncated coverage; they supplied no +new actionable constraint and are not a complete conflict check. No reconciliation +or further CLI debugging was attempted. + +### Repeated live qualification + +At pushed `7b0213bb`, the dedicated coordinator ran: + +```sh +TV_BLAXEL_HARNESS_COORDINATOR=tv-cleanup-coordinator-rose \ +TV_BLAXEL_HARNESS_POOL=rose-cleanup \ +npm test -- local --surface experiment:blaxel-harness +``` + +Canonical run `2026-10-27T21-08-38-267Z-p38866-r679cddb899712621` passed on its +first native attempt in 296.197 seconds, with zero failed or recovered driver +tests, process leaks and infrastructure failures. The retained live attempt is +`.test-runs/blaxel-live-Py7UqI/`; its identity records the configured coordinator, +pool and exact revision, `completed.json` says passed, and `cleanup.json` records +exit 0. The driver retained the default two native retries and two infrastructure +retries. Only its two deliberate abandoned runs used the 60-second interval. + +| Scenario | Canonical child run | Result | +| --- | --- | --- | +| planned retrieval | `2026-10-27T21-09-11-096Z-p39023-r7c42878bb3edd7a5` | expected failed/recovered native fixtures; infrastructure completed | +| held-capacity refusal | `2026-10-27T21-09-44-524Z-p39141-r01321059d8eb93e8` | expected incomplete refusal | +| targeted reuse | `2026-10-27T21-10-33-789Z-p39277-r92cce732e13126b3` | expected failed/recovered native fixtures; infrastructure completed | +| active coordinator killed | `2026-10-27T21-10-49-716Z-p39365-rda3ab840cf51883d` | expected incomplete run | +| active recovery | `2026-10-27T21-11-53-508Z-p39482-r6e6a876c57f088ee` | expected failed/recovered native fixtures; infrastructure completed | +| quiet coordinator killed | `2026-10-27T21-12-07-353Z-p39581-re263e8075af61eb2` | expected incomplete run | +| quiet recovery | `2026-10-27T21-13-11-418Z-p39676-rfa6c95cd577a4d4e` | expected failed/recovered native fixtures; infrastructure completed | +| absent-pool refusal | `2026-10-27T21-13-30-318Z-p39800-ra32edd1d26498fce` | expected incomplete refusal | + +The driver deleted the earlier private pool after its focused reports were retained, +provisioned fresh workers, and repeated every live acceptance. Pool-derived residue +was removed, unrelated sentinels and caches survived, and the neutral mode input +exercised both recovery scenarios. Both private workers were deleted and confirmed +absent at completion. The entire outer run, attempt directory and eight child run +trees are retained locally; the combined log is +`.test-runs/slice-3-static/rose-slice3-r1-live.log`. + +The source correction and proof/document updates are ready for independent +re-review. The dedicated coordinator remains for the later integrated gate. +No tests, builds or static checks ran on the shared host; no shared worker was +changed. Integrated review, target reconciliation, full verification and PR +readiness remain pending. The availability tradeoff on renewal errors and optional +checkout simplification remain as assessed in the review response, without claims +that those refinements were implemented. diff --git a/proofs/arch/test-runner/blaxel-testshards.md b/proofs/arch/test-runner/blaxel-testshards.md index a07843c4..9355135e 100644 --- a/proofs/arch/test-runner/blaxel-testshards.md +++ b/proofs/arch/test-runner/blaxel-testshards.md @@ -56,6 +56,7 @@ The repository-selection assertions: The remaining coverage lives in `test/repo/test-runner-provider.test.ts`. +- The explicit recovery-interval forwarding contract also pins the coordinator's 1200-second default in every timeout profile. This source check proves the configured values; the lease seam and live driver carry actual renewal behavior. - The canonical recommended shard count is 36, and the pool default remains 72 workers using `blaxel/playwright-chromium:latest`, 8192 MB, and the existing labels and naming scheme. - Pool-manager source assertions require pinned nvm `v0.40.3`, `.nvmrc` selection, fresh-worker provisioning before architecture verification, and no alternate image or size defaults. - Coordinator source assertions require nvm activation after checkout and before dependency/test work, and the generated `deps` script includes `.nvmrc` and the root manifest in its dependency cache identity. @@ -83,7 +84,7 @@ The remaining coverage lives in `test/repo/test-runner-provider.test.ts`. - **Contract — cleanup uncertainty.** The cleaner's decision layer consumes authored filesystem-probe results and removal errors. An unexpected mount boundary is skipped, and an incomplete removal is reported without claiming the whole directory was reclaimed; neither permits new test dispatch under [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#Temporary files). These mocked conditions forfeit actual filesystem operations to the cleanup seam and live retrieval assertion; a real mount or full disk adds no necessary coverage. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “mount boundaries and incomplete removal remain reported failures without reclaimed counts” and “cleanup failures keep the old identity, while explicit deletion reserves an expired worker”) ^blaxel-t-scratch-errors -Shared live declaration: the following acceptances run in `experiment:blaxel-harness` via `npm test -- local --surface experiment:blaxel-harness` on a dedicated credentialed Blaxel coordinator sandbox outside the worker pool, from a committed and pushed checkout, never a shared developer host or a leased worker, under [harness placement](../../../specs/arch/test-runner/test-runner.md#^blaxel-harness-placement). They spawn the canonical CLI with real argv and inspect its exit and retained run directory. A fresh dedicated pool, at least two committed native fixture files in `experiment:blaxel-harness-fixtures`, known leftover directories and unrelated sentinels are fixtures. The two files permit the planned run's file-atomic two-shard assignment. Under [surface selection](../../../specs/arch/test-runner/test-registry.md#^blaxel-harness-surfaces), `npm test -- blaxel --tag blaxel-harness-fixture --pool --shards 2 --allow-shard-count-override 1` delegates `--suite selection --surfaces experiment:blaxel-harness-fixtures`; `npm test -- blaxel --surface experiment:blaxel-harness-fixtures --pool ` takes the targeted path. Workers read the committed registry. The coordinator, SDK, worker scripts, native runners, transport, ownership audits and filesystem operations are real. The report-download rendezvous above is the only scheduling hook; no mechanism is mocked. A bounded test driver owns and finally cleans up its coordinators and private workers. These tests do not invoke fixtures restricted to isolated GitHub hosts. +Shared live declaration: the following acceptances run in `experiment:blaxel-harness` via `npm test -- local --surface experiment:blaxel-harness` on a dedicated credentialed Blaxel coordinator sandbox outside the worker pool, from a committed and pushed checkout, never a shared developer host or a leased worker, under [harness placement](../../../specs/arch/test-runner/test-runner.md#^blaxel-harness-placement). The caller supplies `TV_BLAXEL_HARNESS_COORDINATOR` and `TV_BLAXEL_HARNESS_POOL` as described in the [harness instructions](../../../test/runner-fixtures/blaxel-harness/README.md); no personal resource name is prescribed. They spawn the canonical CLI with real argv and inspect its exit and retained run directory. A fresh dedicated pool, at least two committed native fixture files in `experiment:blaxel-harness-fixtures`, known leftover directories and unrelated sentinels are fixtures. The two files permit the planned run's file-atomic two-shard assignment. Under [surface selection](../../../specs/arch/test-runner/test-registry.md#^blaxel-harness-surfaces), `npm test -- blaxel --tag blaxel-harness-fixture --pool --shards 2 --allow-shard-count-override 1` delegates `--suite selection --surfaces experiment:blaxel-harness-fixtures`; `npm test -- blaxel --surface experiment:blaxel-harness-fixtures --pool ` takes the targeted path. Workers read the committed registry. The coordinator, SDK, worker scripts, native runners, transport, ownership audits and filesystem operations are real. The report-download rendezvous above is the only scheduling hook; no mechanism is mocked. A bounded test driver owns and finally cleans up its coordinators and private workers. These tests do not invoke fixtures restricted to isolated GitHub hosts. - **Acceptance — reports before reuse.** A small planned run with at least two shards traverses real acquisition, architecture verification, checkout, runtime/dependency setup, execution and log/report transport under [the shard lifecycle](../../../specs/arch/test-runner/blaxel-testshards.md#Shard lifecycle). Each uploaded script path lies inside its current shard attempt directory. The retained setup phases include `temporary-cleanup`, with cleanup counts and free-space evidence. While it waits at the download rendezvous, observed renewals keep the leases valid; a competing canonical run cannot acquire the occupied capacity or alter its result files. After retrieval, the original native reports, setup/cleanup evidence and diagnostics have the original identities and readable local paths, scratch is gone, and the workers become available to a targeted run. Legacy-prefix and stale-scratch fixtures disappear, unrelated sentinels and persistent caches survive, and the next run does not accumulate an older results tree. The targeted run crosses its distinct dispatch path too. This proves [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release) and [scratch lifetime](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-scratch); the shared diagnostics observations are owned by [reporting](reporting.md#^reporting-t-resource-live). (policy-grade test: `test/runner-fixtures/blaxel-harness/driver/harness.test.ts` “qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle”, with `fixtures/outcomes.test.ts` and `fixtures/temporary.test.ts` under the same harness directory) ^blaxel-t-retrieval diff --git a/proofs/arch/test-runner/test-runner.md b/proofs/arch/test-runner/test-runner.md index 40961e54..d2d5ae0e 100644 --- a/proofs/arch/test-runner/test-runner.md +++ b/proofs/arch/test-runner/test-runner.md @@ -39,7 +39,7 @@ The runner's own tests run the **real `scripts/test/cli.mjs` process** with `spa - Targeted selectors (`--file`, `--surface`, `--package`, `--runner`, `--tag`) and the `--suite unit` / `--suite e2e` / `--suite telemetry-posthog-roundtrip` / `--suite daemon-acceptance` selections pass the guardrail unless a selected surface carries an execution-placement tag. - `daemon-acceptance` fails preflight before its build when `TV_DAEMON_TEST_HOST` is not exactly `1`, and passes that check when the designated-host acknowledgement is present. - `isolated-github-only` surfaces are rejected before preflight for developer local and every Blaxel path; the exact dedicated GitHub workflow identity plus explicit opt-in permits only its local execution path. -- **Acceptance — harness placement configuration.** The real canonical CLI accepts explicitly named coordinator and private-pool environment values for local harness selection without prescribing personal resource names. Missing configuration, an inherited shard owner, and remote driver selection are refused before preflight. Authored environment values and the existing guarded dry-run hook replace only downstream execution; this proves the CLI decision, not actual sandbox placement. The [live harness](blaxel-testshards.md#^blaxel-t-retrieval) checks coordinator labels and crosses the configured private pool. Proves [harness placement](../../../specs/arch/test-runner/test-runner.md#^blaxel-harness-placement). (test to be written: `test/repo/test-runner-guardrails.test.ts`, configured and refused Blaxel harness cases) ^t-blaxel-harness-placement +- **Acceptance — harness placement configuration.** The real canonical CLI accepts explicitly named coordinator and private-pool environment values for local harness selection without prescribing personal resource names. Missing configuration, an inherited shard owner, and remote driver selection are refused before preflight. Authored environment values and the existing guarded dry-run hook replace only downstream execution; this proves the CLI decision, not actual sandbox placement. The [live harness](blaxel-testshards.md#^blaxel-t-retrieval) checks coordinator labels and crosses the configured private pool. Proves [harness placement](../../../specs/arch/test-runner/test-runner.md#^blaxel-harness-placement). (policy-grade tests: `test/repo/test-runner-guardrails.test.ts` “accepts a configured Blaxel harness coordinator and private pool without fixed names” and “refuses missing Blaxel harness configuration and leased or remote placement before preflight”) ^t-blaxel-harness-placement - `--suite all` includes `unit:build-config`, which validates the ordinary CLI output and an isolated release-configured `--outfile` bundle without network or secrets, as proven by [the build seam](../telemetry/sink.md#^t-build-config-integrity). - **Acceptance** (real canonical CLI process, real temporary home with no marker): default `verify` and explicit `--provider auto` select local phases without bypass or Blaxel preflight/fallback diagnostics. The [preflight hook](preflight.md#Test hooks) `TV_TEST_RUNNER_FAKE_REMOTE_PREFLIGHT` is supplied with `TV_TEST_RUNNER_SELFTEST` explicitly removed from the child environment and must remain untouched: reaching remote preflight would fail the command, so a successful local plan proves it was not consulted. The same unauthorized input on a marked home must fail with the remote hook’s authorization error, confirming the detector is active. No remote mechanism is mocked on the unmarked path; `--plan` proves selection and phase construction, not suite execution. Proves [provider resolution](../../../specs/arch/test-runner/test-runner.md#Verify orchestration). Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-local-verify-default - **Acceptance** (real CLI process and temporary marked home; remote checks replaced by the declared preflight hook): a passing remote result makes default `verify --plan` select Blaxel phases. This proves provider selection, forfeiting real service and Git checks to the [preflight proof](preflight.md#Assertions), which records their committed boundary-test gap. The [live-evidence directive](../../../specs/arch/test-runner/preflight.md#Testing) governs live validation. Passing self-test results are refused outside `verify --plan`, before any remote dispatch. Proves [provider resolution](../../../specs/arch/test-runner/test-runner.md#Verify orchestration). Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-marked-verify-default From 7c66d160c2494bb673e16010563f0c8b5887f36e Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:29:47 +0000 Subject: [PATCH 54/64] Record slice convergence and integrated harness reconciliation --- ...est-harness-resource-cleanup-integrated.md | 87 +++++++++++++++++++ .../test-harness-resource-cleanup-plan.md | 4 +- ...arness-resource-cleanup-review-response.md | 16 +++- ...ness-resource-cleanup-slice-3-review-r2.md | 62 +++++++++++++ .../test-harness-resource-cleanup-slice-3.md | 2 +- test/repo/test-runner-guardrails.test.ts | 2 + 6 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 docs/working/test-harness-resource-cleanup-integrated.md create mode 100644 docs/working/test-harness-resource-cleanup-slice-3-review-r2.md diff --git a/docs/working/test-harness-resource-cleanup-integrated.md b/docs/working/test-harness-resource-cleanup-integrated.md new file mode 100644 index 00000000..b4591aed --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-integrated.md @@ -0,0 +1,87 @@ +# Test-harness resource cleanup: integrated validation + +Status: all three slices converged; target reconciliation complete; full gate pending. +Owner: rose. Draft PR #12. Branch: `thopter/test-harness-resource-cleanup`. +No test, build, lint, type check, browser or server runs on the shared development +host. The dedicated Blaxel coordinator performs all execution below. + +## Target reconciliation + +On 2026-10-27, fetched `origin` and read the complete branch histories on both sides. +`origin/main` is `a8e73fbf5eecfb27172ff5928ffb71f0c6844368`, already an ancestor +of the reviewed branch head `dc1d9f2e`. The merge base is that same main commit; +there are no incoming commits to merge. The earlier real merge is `74d22941`. +The logical-merging skill was read in full. Its policy-level reconciliation still +applies; an artificial merge adds no integration evidence. + +Audited the complete changed-path inventory against main and the governing specs, +proofs and converged slices. There are no production application changes. Checked +the interaction paths beyond changed files: + +- Worker build ownership spans shared Playwright/Vitest fixtures and direct + product-server/versioned-bundle callers. Failed preparation is removed; pending + preparation and servers settle before reusable worker outputs are disposed. + Version-keyed reuse and serialized web builds remain intact. +- The five built-application conversions preserve same-origin authentication and + stable-front restart continuity. PR #9's built reset and the separate Settings + development walk remain. Main's demo fixture connection teardown remains. +- The Vite service still strips conditional validators before its middleware, + preserving main's full-response correction. This contribution claims no measured + incremental memory saving after that correction. +- Parent diagnostics and bounded retained links cross planned and targeted paths, + provider normalization, retry aggregation and final reporting. Owner attribution + and `TMPDIR` survive restricted child environments. +- Reservation, dispatch registration, teardown, retrieval, recovery and pool + publication/deletion retain the converged exclusion rules. The default recovery + interval is 1200 seconds for every profile. Existing execution deadlines and + native/infrastructure retry budgets remain. Main's origin-repository checkout + selection is unchanged; result-reset preservation remains covered by slice 3. +- The two harness experiment surfaces are excluded from ordinary `all` and CI. + The committed private driver and every new live proof obligation have evidence; + no new pending proof citation remains. Intentional fixture failures are retained + as failures, separate from the passing outer qualification. + +No textual conflict, policy conflict or additional behavior edit was found. The +only code-file change after slice convergence is the review-requested dry-run +comment. The full gate validates the composition at the pushed checkpoint. + +## Slice 3 review observations + +[Round 2](test-harness-resource-cleanup-slice-3-review-r2.md) passes the slice. A +comment now explains that clearing the inherited shard token is limited to the +two immediate dry-run CLI placement cases; ordinary surface ownership remains. +Record both the number of shard attempts and distinct shard indices reporting +`Lease renewal lost` in the full-gate result. Fail-on-first-renewal-error behavior +is unchanged; do not infer transport tolerance from the private qualification. + +## Private cleanup and full gate + +The passing live driver deleted `tv-testshard-x64-rose-cleanup-01` and `-02` through +the canonical pool manager. All its evidence is retained locally. Read-only API +inventory at 2026-10-27 21:26:06 UTC finds only `tv-cleanup-coordinator-rose` among +those three private resources. It reports 144 deployed sandboxes overall; this is +an inventory, not a quota or active-concurrency measurement. + +Before the gate, confirm private-pool deletion and absence through the canonical +manager on the coordinator. Keep that coordinator outside the worker-pool labels. +At the fixed committed and pushed integration checkpoint, run: + +```sh +npm test -- pool blaxel delete --pool rose-cleanup +npm test -- pool blaxel list --pool rose-cleanup +npm run verify -- blaxel --pool poc +``` + +The gate uses all standard 36 shards and the default retry budgets. No fault +experiment, provisioning or deletion targets shared workers. Retain the final +summary, attempts, diagnostics and coordinator log before reporting. Ordinary +lease acquisition/cleanup is authorized by the plan, including clean expired +legacy residue; uncertain ownership must hold and report. + +## Remaining work and limits + +Full-gate results and renewal-loss counts are pending. Integrated independent +review follows the result. Docs preparation, PR-ready status, Josh's spec review +and required GitHub checks remain later obligations. No new lasting decision is +made in this integration step. Primitive's earlier unread warning remains in the +slice record; incomplete hook scans are not a complete governing check. diff --git a/docs/working/test-harness-resource-cleanup-plan.md b/docs/working/test-harness-resource-cleanup-plan.md index 54ca8b01..e9e0a7f2 100644 --- a/docs/working/test-harness-resource-cleanup-plan.md +++ b/docs/working/test-harness-resource-cleanup-plan.md @@ -1,6 +1,6 @@ # Test-harness resource cleanup: implementation plan -Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; [slice 1 has converged](test-harness-resource-cleanup-slice-1-review-r1.md), and [slice 2 has converged](test-harness-resource-cleanup-slice-2-review-r1.md). [Slice 3](test-harness-resource-cleanup-slice-3.md) is implemented and privately qualified, awaiting independent review. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. +Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; [slice 1 has converged](test-harness-resource-cleanup-slice-1-review-r1.md), and [slice 2 has converged](test-harness-resource-cleanup-slice-2-review-r1.md). [Slice 3 has converged](test-harness-resource-cleanup-slice-3-review-r2.md) at `dc1d9f2e`. [Integrated validation](test-harness-resource-cleanup-integrated.md) is in progress. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. ## Slice judgment @@ -93,4 +93,4 @@ Retrieve all evidence before deleting the dedicated coordinator, and report clea ## Current limits -Slice 1's helper contracts, real framework teardown and five built-application walks have converged. Slice 2's diagnostics have converged. Slice 3's focused contracts, Linux seams and live private qualification have passed; its evidence record carries the failed attempts, fixes, final revisions and retained run paths. Both private workers were deleted through the manager and confirmed absent; the dedicated coordinator remains for integrated verification. Slice 3 and integrated review, target reconciliation and the full gate are still outstanding. Provider inventory differs from the historical pool count; account-specific quota, idle-sandbox accounting, the cause of one deliberately shortened lease's timing lapse, and full-gate duration remain unknown. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. +Slice 1's helper contracts, real framework teardown and five built-application walks have converged. Slice 2's diagnostics have converged. Slice 3's focused contracts, Linux seams and live private qualification have passed; its evidence record carries the failed attempts, fixes, final revisions and retained run paths. Both private workers were deleted through the manager and confirmed absent; the dedicated coordinator remains for integrated verification. Slice 3 has converged. Target reconciliation found no incoming main commits; integrated review and the full gate remain outstanding. Provider inventory differs from the historical pool count; account-specific quota, idle-sandbox accounting, the cause of one deliberately shortened lease's timing lapse, and full-gate duration remain unknown. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 7caa7de4..5f3fa38e 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -1,6 +1,6 @@ # Resource-cleanup proposal: review response -Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, proof round 2 at `3fa15d47`, and plan round 2 at `04a6b275` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. Slices 1 and 2 have converged; slice 3 round 1 is corrected and ready for re-review. Earlier sections retain the response history, including the superseded private full-gate arrangement. Reviewer: Claude Fable (Nina), arranged by the supervisor. +Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, proof round 2 at `3fa15d47`, and plan round 2 at `04a6b275` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. All three implementation slices have converged; integrated validation is in progress. Earlier sections retain the response history, including the superseded private full-gate arrangement. Reviewer: Claude Fable (Nina), arranged by the supervisor. Independent round 1 reviewed `1652787e`. The [full review](test-harness-resource-cleanup-review-r1.md) remains unchanged: FAIL on two blockers. The response committed at `99b7ed93` narrowed the work pending Josh's decisions. The following sections through round 2 record that history, not the current review verdict. @@ -323,3 +323,17 @@ then passed on its first attempt in 4m 56s with fresh private workers. Both work were deleted and confirmed absent; the complete reports are retained. No shared host tests or shared-pool experiments ran. The correction is ready for re-review; the subsequent documentation/proof commit changes no executable behavior. + + +## Slice 3 round 2: convergence and integrated validation + +The [complete review](test-harness-resource-cleanup-slice-3-review-r2.md) passes +`dc1d9f2e` with no blocker. Both observations are accepted under the bounded rule: +add one comment explaining why the two immediate CLI dry runs deliberately clear +the inherited shard token, and count shards that report `Lease renewal lost` in +the ordinary 36-shard gate. No lifecycle, default or retry policy changes follow. + +The [integrated record](test-harness-resource-cleanup-integrated.md) carries the +current-target reconciliation and full-gate evidence. The supervisor will arrange +integrated review after the gate result. Docs preparation and marking PR #12 ready +remain later steps. diff --git a/docs/working/test-harness-resource-cleanup-slice-3-review-r2.md b/docs/working/test-harness-resource-cleanup-slice-3-review-r2.md new file mode 100644 index 00000000..d1c600e3 --- /dev/null +++ b/docs/working/test-harness-resource-cleanup-slice-3-review-r2.md @@ -0,0 +1,62 @@ +# Slice 3 implementation review, round 2: test-harness resource cleanup (PR #12, dc1d9f2e) + +**Verdict: PASS.** The round 1 blocker is resolved at its root: the canonical CLI now admits `experiment:blaxel-harness` only with a non-empty coordinator name and pool supplied by the caller, local execution and no inherited shard token; the driver takes both names from the same variables, keeps the real label check on the named sandbox, derives its leftover fixtures and sentinels from the pool, and reads a neutral per-worker mode file; no personal resource name remains anywhere under `scripts/`, `test/`, the registry, the proofs or the specs. The CLI decision has its own red-then-green acceptance against the real process, the proofs name it, a README states how the qualification is invoked and what it destroys, and the complete live driver ran again in its committed form on fresh private workers and passed on its first attempt. The lease, recovery, cleanup, pool-manager and coordinator code is byte-identical to the round 1 revision (`git diff --stat 2dc0e7d6 dc1d9f2e -- scripts/` touches only the four CLI lines), so everything round 1 established about the protocol stands, and I re-read the whole slice against the proofs rather than only the correction. Refinement 2 is applied; refinements 1 and 3 are declined or deferred with reasons I accept below. No blockers; two observations, neither requiring a change before the integrated review. + +- Reviewed: `git diff 2dc0e7d6 dc1d9f2e` in full (CLI guard, driver, fixture support, the two guardrail cases, the profile pin, the README, two proof files, the plan line, the response and the record); the current driver, fixtures, lease module, lifecycle helpers, pool lifecycle, coordinator and manager as they now stand; the plan's slice 3; the Blaxel, placement and registry specs; the testing policy and the reviewer checklist; plus the new run directories, the new live attempt directory and the three new static logs in this worktree. +- Branch state: contains `origin/main` (`a8e73fbf`); the copied round 1 review is identical to mine; `dc1d9f2e` changes only documents and proofs on top of the executable checkpoint `7b0213bb`. +- I ran nothing. I read summaries, native reports, receipts, sidecars, placement and cleanup records, manager logs and the live attempt records with small scripts; nothing was built, tested or changed. + +## The blocker, resolved + +`scripts/test/cli.mjs:870-874` refuses the driver surface unless the provider is `local`, `TV_TEST_SHARD_OWNER` is absent, and both `TV_BLAXEL_HARNESS_COORDINATOR` and `TV_BLAXEL_HARNESS_POOL` are non-blank; the message names the two variables. `harness.test.ts:10-11` reads the same two values, `:104` refuses to start without them or under a shard token, and `:105-106` still fetches the named sandbox through the SDK and requires it not to carry the worker label, which is the check that actually distinguishes a coordinator from a pool worker. The leftover fixtures, stale scratch and both sentinels are named from the pool (`:17-23`, seeded at `:117-123`, asserted at `:168-169`); the mode file is `/tmp/tv-blaxel-harness-mode.json` on both sides (`:17`, `fixtures/support.ts:7`), written by the driver to each fresh worker and gone with it, so no production environment forwarding was added to carry fixture input. `test/runner-fixtures/blaxel-harness/README.md` says where the driver runs, what it requires, that it deletes the named pool before and after, and that the coordinator is the caller's to dispose of. The plan's invocation line now passes the names as configuration and says they are this contribution's resources, not the driver's. + +**Placement acceptance (`^t-blaxel-harness-placement`, `test/repo/test-runner-guardrails.test.ts:740-768`).** The real CLI, with authored environment values and the existing guarded dry-run hook, exits 0 and plans `surfaces=experiment:blaxel-harness` when both names are set; a blank coordinator, a missing pool, an inherited shard owner and `blaxel` as provider each exit 2 with the placement message and no `[dry-run]` output. The proof says what this proves (the CLI decision) and what it does not (sandbox identity, which the live driver's label check and configured pool carry). Red at `6d505597`, where the real CLI still demanded the literal and returned 2 under the neutral names; green at `7b0213bb`. + +**Live qualification repeated (`.test-runs/blaxel-live-Py7UqI/`).** `identity.json` records the configured coordinator, the configured pool and revision `7b0213bb`; `completed.json` is `passed` with all eight run pointers; `cleanup.json` is exit 0. The driver deleted the earlier pool and provisioned two fresh workers (`initial-delete.log`, `ensure.log`), the held deletion was refused on both workers while the planned run waited at the download gate (`held-delete.log`), the competing run acquired neither, the final deletion removed both and a fresh dispatch failed on the empty pool. Both workers' `temporary-cleanup.json` on the planned run report exactly one directory per prefix and one stale scratch directory under the pool-derived names; the recovered runs carry two receipts each, now 1,061 bytes with the worker name in `previous.worker`, eight attributed processes with SIGTERM and SIGKILL outcomes in the active scenario and 491-byte receipts with empty process lists in the quiet one; the abandoned leases show the 60-second interval and the recovered leases the 1,200-second default; failed-surface sidecars are 1,737–1,787 bytes with `kernel: available`; the placement records show sixteen Chromium descriptors, four unlinked, under the attempt's `tmp/`, Chromium 148.0.7778.96. The outer run passed its single native attempt in 295.1 s (summary 296.2 s) with zero failed or recovered tests, leaks or infrastructure failures, and lint and type-check passed at the same revision. + +## Refinement dispositions + +1. **Renewal stops on the first failed exec: declined, accepted.** The response keeps the fail-closed behavior and states the cost plainly: one failed renewal fails the shard as infrastructure and holds the worker until ordinary recovery. The reasoning is sound as far as it goes (a failed exec gives no evidence either way, and a safe retry needs the last confirmed deadline and new failure coverage), the behavior is correct rather than merely tolerable, and the proof's lease-ordering contract already names it. This is an availability trade-off the owner may make. It will be measured for the first time at the 36-shard integrated gate, where the number of renewal execs is two orders of magnitude larger than in this slice's runs; see observation 1. +2. **Profile-independent default: applied.** `test-runner-provider.test.ts:34-37` requires `lockTtlSeconds: 1200` for `normal`, `cold` and `repair` in the coordinator source, and the proof's provider-file list names the assertion as a source check with the seam and driver carrying behavior. +3. **Checkout reset: deferred, accepted.** Optional in round 1; the reason given (it touches repository switching, whose spec requires live two-repository evidence) is the one I anticipated, and the preservation path it would replace is covered and passed. + +## The whole slice, re-read + +Nothing in the lease state machine (`scripts/test/blaxel-worker-lease.mjs`), the coordinator lifecycle (`blaxel-lease-lifecycle.mjs`, `run-blaxel-testshards.mjs`), the pool lifecycle or the manager changed, so round 1's findings on them hold: one serialized worker-side function with nine operations, recovery as the single expired branch of acquisition reusing normal teardown's stop, audit and clean, coordinator-only renewal, dispatch registered before the API request and entry refused after ownership changes, release only after confirmed teardown and retained results, fail-closed holds on unregistered dispatch, unclean audits, cleanup errors and lost renewal, and pool publication after verification with deletion reserving before unpublishing. The eleven focused obligations and the registry contract still cite tests that passed on the private pool at `45202ee9`, and the only test files touched since then (`test-runner-guardrails.test.ts`, `test-runner-provider.test.ts`) passed whole at `7b0213bb` (65 passed with the one existing GitHub-only skip; 43 passed). The five live acceptances and `^reporting-t-resource-live` now cite a driver that has run in its committed form. + +## The record, checked + +| Claim | Evidence | +| --- | --- | +| `6d505597` intended red under `--grep`; four green rows at `7b0213bb` with the stated counts | Red run `testsFailed 1`, infrastructure completed, naming the "accepts a configured…" case; green runs: 2 of 66 under the grep, the provider grep passed, 65 passed and 1 skipped whole, 43 passed whole; zero recovered retries, leaks and infrastructure failures throughout. | +| Lint and type-check at `7b0213bb` | `rose-slice3-r1-green.log` ends with both clean, after summaries naming that commit. | +| Live run first attempt, 296.197 s, zero failures, retries, leaks, infrastructure failures | Attempt 0 passed in 295,131 ms; `summary.json` 296,197 ms at `7b0213bb`; counts all zero. | +| Eight child runs with the tabulated outcomes | All eight present at `7b0213bb`: four `failed/completed` with one failure and one recovered, four `incomplete` (two refusals, two killed coordinators). | +| Pool deleted, re-provisioned fresh, deleted again and confirmed absent | `initial-delete.log`, `ensure.log` (both workers provisioned to `DEPLOYED`), `final-delete.log`; the empty-listing and absent-pool assertions are in the passing driver. | +| 60-second interval only on the two abandoned runs | Driver source `:49-51`, `:181`; recovery evidence shows the abandoned leases at 60 s and the recovered runs' leases at 1,200 s. | +| No shared host, no shared pool | Every coordinator line names `--pool rose-cleanup`; the driver ran under `local` on the coordinator. | + +Not verifiable here: the 144-sandbox inventory and the Primitive hook behavior the record describes. + +## Observations (none blocking) + +1. **Report renewal losses at the integrated gate.** The declined refinement means a transport failure on any of the several hundred renewal execs a 36-shard run makes will show up as an infrastructure-failed shard, a retry on another worker and one `poc` worker held for twenty minutes. That is acceptable and spec-conformant, but it is the first time the choice will be exercised at scale; the integrated record should say how many shards, if any, failed with "Lease renewal lost" so Josh can judge the trade-off on evidence rather than on this slice's small runs. +2. **The guardrail cases clear the shard token deliberately.** `TV_TEST_SHARD_OWNER: undefined` in both new cases is what lets the real CLI pass on a Blaxel worker, where the test process itself carries the outer token; the spawned CLI is a dry run that exits at once, and the runs report no leaks. Worth one comment in the test so the next reader does not take it for an accidental weakening of ownership attribution. + +## Reviewer checklist + +- **Proof coverage.** Every slice obligation cites a test that passed on the private pool at the revision it names; the placement acceptance is new, cited, and red-then-green; the live citations name the driver that ran at `7b0213bb`. +- **Mocking and boundaries.** Unchanged from round 1 for the protocol; the new acceptance substitutes only the declared dry-run hook and authored environment values and says so. +- **Honesty of evidence.** The record distinguishes the intended red, the green rows and the repeated qualification, keeps the original history under a renamed heading, and claims nothing about the declined refinement beyond its stated cost. +- **Scope.** Four CLI lines, the driver and fixture input, two tests, a README, proofs and working documents; no production file; no lease, retry, timeout or pool change. +- **Baseline.** Green for the slice's obligations; integrated review, target reconciliation, the `poc` gate at 36 shards and pre-merge docs preparation remain, as the plan and record state. + +## Evidence and limits + +- Read every changed file and the current driver and fixtures in full; scanned `scripts/`, `test/`, the registry, the proofs and the specs for the personal names; cross-checked the sixteen new run summaries, the native reports behind the whole-file counts, the live attempt's identity, completion, cleanup and manager logs, both recovery-evidence files, all receipts, sidecars, placement and cleanup records of the four completed child runs, and the three new static logs. +- Not established: anything about the private workers beyond the downloaded files; the renewal-loss rate under a full-size run. + +## Housekeeping + +- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel beyond scripts that read files already on disk. `git fetch origin` updated remote-tracking refs. +- Primitive: the record reports that its commit-hook scans again returned truncated results and no new constraint, and that the earlier warning remains unread as the supervisor directed. In my session the start hook's connection notice and `prim` instruction were the only Primitive activity; a review establishes no lasting decision, so I did not invoke it, and nothing from Primitive reached or shaped this review. diff --git a/docs/working/test-harness-resource-cleanup-slice-3.md b/docs/working/test-harness-resource-cleanup-slice-3.md index 114bb0ff..54283204 100644 --- a/docs/working/test-harness-resource-cleanup-slice-3.md +++ b/docs/working/test-harness-resource-cleanup-slice-3.md @@ -1,6 +1,6 @@ # Slice 3: exclusive Blaxel worker lifecycle -Status: review round 1 corrected and privately qualified; ready for independent re-review. +Status: [review round 2](test-harness-resource-cleanup-slice-3-review-r2.md) passed at `dc1d9f2e`; slice 3 has converged. Integrated validation is recorded [separately](test-harness-resource-cleanup-integrated.md). Final executable checkpoint: `7b0213bb`. The repeated live qualification passed on its first native attempt in 4m 56s. The correction and evidence follow the original qualification history below. diff --git a/test/repo/test-runner-guardrails.test.ts b/test/repo/test-runner-guardrails.test.ts index f244d30d..b1af33eb 100644 --- a/test/repo/test-runner-guardrails.test.ts +++ b/test/repo/test-runner-guardrails.test.ts @@ -737,6 +737,8 @@ describe("test runner guardrails", () => { expect(result.stderr).toContain("--provider must be one of: local, blaxel"); }); + // These two placement cases clear the outer shard token only for an immediate + // CLI dry run; the child retains ordinary surface ownership and starts no work. test("accepts a configured Blaxel harness coordinator and private pool without fixed names", () => { const result = runCli(["local", "--surface", "experiment:blaxel-harness"], selftestEnv({ TV_TEST_RUNNER_DRY_RUN: "1", From 30774259812fb5457ed7a248d8123af3ee24ded7 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:36:56 +0000 Subject: [PATCH 55/64] Record passing shared-pool full gate and private cleanup --- ...est-harness-resource-cleanup-integrated.md | 94 ++++++++++++++++--- .../test-harness-resource-cleanup-plan.md | 4 +- ...arness-resource-cleanup-review-response.md | 19 +++- 3 files changed, 100 insertions(+), 17 deletions(-) diff --git a/docs/working/test-harness-resource-cleanup-integrated.md b/docs/working/test-harness-resource-cleanup-integrated.md index b4591aed..8e47efbb 100644 --- a/docs/working/test-harness-resource-cleanup-integrated.md +++ b/docs/working/test-harness-resource-cleanup-integrated.md @@ -1,6 +1,6 @@ # Test-harness resource cleanup: integrated validation -Status: all three slices converged; target reconciliation complete; full gate pending. +Status: all three slices converged; target reconciliation and the full gate passed. Ready for independent integrated review. Owner: rose. Draft PR #12. Branch: `thopter/test-harness-resource-cleanup`. No test, build, lint, type check, browser or server runs on the shared development host. The dedicated Blaxel coordinator performs all execution below. @@ -62,9 +62,9 @@ inventory at 2026-10-27 21:26:06 UTC finds only `tv-cleanup-coordinator-rose` am those three private resources. It reports 144 deployed sandboxes overall; this is an inventory, not a quota or active-concurrency measurement. -Before the gate, confirm private-pool deletion and absence through the canonical -manager on the coordinator. Keep that coordinator outside the worker-pool labels. -At the fixed committed and pushed integration checkpoint, run: +At pushed checkpoint `7c66d160c2494bb673e16010563f0c8b5887f36e`, the canonical +private-pool delete and list both exited zero with no workers listed. The dedicated +coordinator remained outside the worker-pool labels and then executed: ```sh npm test -- pool blaxel delete --pool rose-cleanup @@ -72,16 +72,82 @@ npm test -- pool blaxel list --pool rose-cleanup npm run verify -- blaxel --pool poc ``` -The gate uses all standard 36 shards and the default retry budgets. No fault -experiment, provisioning or deletion targets shared workers. Retain the final -summary, attempts, diagnostics and coordinator log before reporting. Ordinary -lease acquisition/cleanup is authorized by the plan, including clean expired -legacy residue; uncertain ownership must hold and report. +The gate used all standard 36 shards and the default native/infrastructure retry +budgets of two. No fault experiment, provisioning or deletion targeted shared +workers. Normal lease acquisition and cleanup ran under the converged rules. + +## Full-gate result + +`npm run verify -- blaxel --pool poc` exited **0** at `7c66d160`: + +- All 36 shard indices passed on infrastructure attempt 1. All 22 selected + surfaces ran and passed, covering 370 assigned files. Aggregated native counts: + **3,705 passed, 8 skipped, 0 failed, 0 recovered retries** (3,713 total). +- **0 process leaks and 0 infrastructure failures.** `Lease renewal lost`: + **0 shard attempts, 0 distinct shard indices**, checked against the complete + provider report, attempt records and logs. This one run does not establish a + transport-error tolerance guarantee or a long-term renewal failure rate. +- Lint, type checking, package manifests and the final vibe-mode check passed. + `verify:total` was 141,073 ms; the test run was 119,945 ms. Test execution ran + from 21:30:30.831 to 21:32:30.776 UTC on 2026-10-27. +- Retained 231 resource sidecars, all linked from the final surface results and + present on disk, each with observed start/end samples. Every OOM-kill delta was + zero. There were no native failed attempts, so kernel failure excerpts were + correctly not requested. Largest sidecar: 1,741 bytes; largest combined resource + sidecars for one shard: 13,727 bytes. These are interval observations, not peaks. +- The 36 pre-run cleanup reports show 1,013 source-CLI directories and 497 web + bundle directories removed, with no cleanup skips/errors. Their recorded + allocated bytes sum to **5,003,157,504 bytes** (about 5.0 GB / 4.66 GiB), equal + to the summed before/after available-space delta. This is actual legacy residue + reclaimed during ordinary acquisition, not a seeded shared-worker experiment. + +Four acquisition candidates were skipped because +`/home/playwright/.nvm/nvm.sh` was absent: `tv-testshard-x64-poc-101`, `-08`, `-109` +and `-12`. The runner acquired replacements and every planned shard completed. +Those skipped workers were not repaired or reconfigured. Their missing runtime +cause remains unknown and is independent of the passing selected-shard results. + +Automatic attestation publication was attempted after verification and failed +non-fatally: GitHub returned HTTP 403, permission denied to `jwhiting`, during the +ref update/fallback fetch. **The gate passed; no attestation publication is +claimed.** No credential changes, attestation retry or bypass was attempted. + +A fresh `git fetch origin` after the gate still resolves main to `a8e73fbf`. +The tested tree contains that full target. Subsequent record edits change only +working documents; `7c66d160` is the exact tested revision, not an assertion that +any later documentation commit ran another gate. + +## Retained evidence + +The complete run is +[`.test-runs/2026-10-27T21-30-30-229Z-p40561-rde511175732d79a8/summary.json`](../../.test-runs/2026-10-27T21-30-30-229Z-p40561-rde511175732d79a8/summary.json). +Its `results.json`, `provider/blaxel/report.json`, provider summary, all 36 shard +logs/reports, native output and cleanup/resource sidecars are retained beneath +that directory. SDK binary downloads copied **1,100 files / 23,423,163 bytes**; +each downloaded length matched the coordinator filesystem inventory. + +Additional local evidence is under `.test-runs/integrated-validation/`: +`rose-integrated-verify.log` (including private-pool absence and the attestation +error), `download-inventory.json`, `metrics.json`, and `private-cleanup.json`. +No test or static check executed on the shared development host. + +After retaining evidence, the coordinator's API listed every dispatched process +as completed or failed, with the full gate completed/exit 0. Requested deletion +of the dedicated coordinator only. Read-only API inventory at +**2026-10-27 21:36:31 UTC** confirms all three private resource names absent +(143 remaining sandboxes, all deployed). The two private workers had already +been deleted. No shared worker was deleted. ## Remaining work and limits -Full-gate results and renewal-loss counts are pending. Integrated independent -review follows the result. Docs preparation, PR-ready status, Josh's spec review -and required GitHub checks remain later obligations. No new lasting decision is -made in this integration step. Primitive's earlier unread warning remains in the -slice record; incomplete hook scans are not a complete governing check. +Independent integrated review is next. Docs preparation, PR-ready status, Josh's +spec review and required GitHub checks remain later obligations. The attestation +write did not succeed. The four skipped workers' runtime problem is unresolved. +Incremental memory savings after the Vite correction and long-term renewal-loss +frequency remain outside this evidence. + +No new lasting decision was made in this integration step. Primitive's commit +hook listed eight existing decisions, including superseded conversation wording, +and reported a truncated scan. This is not a complete governing check; the user +corrections and converged specs remain authoritative. The earlier unread warning +remains in the slice record; its CLI received no further debugging. diff --git a/docs/working/test-harness-resource-cleanup-plan.md b/docs/working/test-harness-resource-cleanup-plan.md index e9e0a7f2..afead883 100644 --- a/docs/working/test-harness-resource-cleanup-plan.md +++ b/docs/working/test-harness-resource-cleanup-plan.md @@ -1,6 +1,6 @@ # Test-harness resource cleanup: implementation plan -Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; [slice 1 has converged](test-harness-resource-cleanup-slice-1-review-r1.md), and [slice 2 has converged](test-harness-resource-cleanup-slice-2-review-r1.md). [Slice 3 has converged](test-harness-resource-cleanup-slice-3-review-r2.md) at `dc1d9f2e`. [Integrated validation](test-harness-resource-cleanup-integrated.md) is in progress. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. +Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; [slice 1 has converged](test-harness-resource-cleanup-slice-1-review-r1.md), and [slice 2 has converged](test-harness-resource-cleanup-slice-2-review-r1.md). [Slice 3 has converged](test-harness-resource-cleanup-slice-3-review-r2.md) at `dc1d9f2e`. [Integrated validation](test-harness-resource-cleanup-integrated.md) passed; independent integrated review is next. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. ## Slice judgment @@ -93,4 +93,4 @@ Retrieve all evidence before deleting the dedicated coordinator, and report clea ## Current limits -Slice 1's helper contracts, real framework teardown and five built-application walks have converged. Slice 2's diagnostics have converged. Slice 3's focused contracts, Linux seams and live private qualification have passed; its evidence record carries the failed attempts, fixes, final revisions and retained run paths. Both private workers were deleted through the manager and confirmed absent; the dedicated coordinator remains for integrated verification. Slice 3 has converged. Target reconciliation found no incoming main commits; integrated review and the full gate remain outstanding. Provider inventory differs from the historical pool count; account-specific quota, idle-sandbox accounting, the cause of one deliberately shortened lease's timing lapse, and full-gate duration remain unknown. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. +Slice 1's helper contracts, real framework teardown and five built-application walks have converged. Slice 2's diagnostics have converged. Slice 3's focused contracts, Linux seams and live private qualification have passed; its evidence record carries the failed attempts, fixes, final revisions and retained run paths. Both private workers were deleted through the manager and confirmed absent. Slice 3 has converged. Target reconciliation found no incoming main commits. The standard 36-shard full gate passed at `7c66d160` with no failed/recovered tests, leaks, infrastructure failures or renewal losses; integrated review remains outstanding. The integrated record names eight native skips, four skipped acquisition candidates, the non-fatal attestation publication error, retained evidence and final private-coordinator cleanup. Provider inventory differs from the historical pool count; account-specific quota, idle-sandbox accounting and the cause of one deliberately shortened lease's timing lapse remain unknown. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md index 5f3fa38e..08919443 100644 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ b/docs/working/test-harness-resource-cleanup-review-response.md @@ -1,6 +1,6 @@ # Resource-cleanup proposal: review response -Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, proof round 2 at `3fa15d47`, and plan round 2 at `04a6b275` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. All three implementation slices have converged; integrated validation is in progress. Earlier sections retain the response history, including the superseded private full-gate arrangement. Reviewer: Claude Fable (Nina), arranged by the supervisor. +Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, proof round 2 at `3fa15d47`, and plan round 2 at `04a6b275` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. All three implementation slices have converged; the integrated gate passed and independent integrated review is next. Earlier sections retain the response history, including the superseded private full-gate arrangement. Reviewer: Claude Fable (Nina), arranged by the supervisor. Independent round 1 reviewed `1652787e`. The [full review](test-harness-resource-cleanup-review-r1.md) remains unchanged: FAIL on two blockers. The response committed at `99b7ed93` narrowed the work pending Josh's decisions. The following sections through round 2 record that history, not the current review verdict. @@ -337,3 +337,20 @@ The [integrated record](test-harness-resource-cleanup-integrated.md) carries the current-target reconciliation and full-gate evidence. The supervisor will arrange integrated review after the gate result. Docs preparation and marking PR #12 ready remain later steps. + + +## Integrated checkpoint: full gate passed + +The [integrated record](test-harness-resource-cleanup-integrated.md) reports the +whole-tree reconciliation against current main `a8e73fbf` and the standard +36-shard gate at pushed `7c66d160`. It passed: 22 surfaces, 3,705 tests passed, +eight skipped, zero failed or recovered tests, zero process leaks or infrastructure +failures, and zero renewal-loss shard attempts. Static verification also passed. +All results are retained locally; private infrastructure cleanup is recorded. + +Automatic attestation publication failed non-fatally with GitHub HTTP 403. Four +worker candidates without the configured nvm runtime were skipped and replaced; +they were not repaired. Neither issue is hidden by the passing gate. The exact +tested commit is `7c66d160`; this following working-record update runs no tests. +The contribution now stops for the supervisor's independent integrated review. +Docs preparation and marking PR #12 ready have not begun. From 278e906e6430c3c8aa0775b54a80f124359305d2 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:50:43 +0000 Subject: [PATCH 56/64] Apply bounded refinements from integrated harness review --- ...est-harness-resource-cleanup-integrated.md | 20 +++++++++++++++++-- proofs/arch/test-runner/sharded-execution.md | 2 +- scripts/run-blaxel-testshards.mjs | 2 ++ scripts/test/blaxel-worker-lease.d.mts | 2 -- scripts/test/blaxel-worker-lease.mjs | 3 --- 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/docs/working/test-harness-resource-cleanup-integrated.md b/docs/working/test-harness-resource-cleanup-integrated.md index 8e47efbb..c39e0caa 100644 --- a/docs/working/test-harness-resource-cleanup-integrated.md +++ b/docs/working/test-harness-resource-cleanup-integrated.md @@ -1,6 +1,6 @@ # Test-harness resource cleanup: integrated validation -Status: all three slices converged; target reconciliation and the full gate passed. Ready for independent integrated review. +Status: independent integrated review passed at `30774259`; the contribution has converged. Pre-merge docs preparation and ready-PR CI follow. Owner: rose. Draft PR #12. Branch: `thopter/test-harness-resource-cleanup`. No test, build, lint, type check, browser or server runs on the shared development host. The dedicated Blaxel coordinator performs all execution below. @@ -110,7 +110,7 @@ cause remains unknown and is independent of the passing selected-shard results. Automatic attestation publication was attempted after verification and failed non-fatally: GitHub returned HTTP 403, permission denied to `jwhiting`, during the ref update/fallback fetch. **The gate passed; no attestation publication is -claimed.** No credential changes, attestation retry or bypass was attempted. +claimed.** Ready-PR CI must therefore run its heavy jobs in full rather than reuse this Blaxel result through a `refs/testpass` attestation. No credential changes, attestation retry or bypass was attempted. A fresh `git fetch origin` after the gate still resolves main to `a8e73fbf`. The tested tree contains that full target. Subsequent record edits change only @@ -151,3 +151,19 @@ hook listed eight existing decisions, including superseded conversation wording, and reported a truncated scan. This is not a complete governing check; the user corrections and converged specs remain authoritative. The earlier unread warning remains in the slice record; its CLI received no further debugging. + + +## Integrated review refinements + +The independent integrated review passed at `30774259`. Removed the two unused +lease-module constants and their declarations, corrected the stale pending-proof +sentence, and commented the release helper's interruption flag without changing +its behavior. The attestation paragraph now states the consequence for ready-PR +CI. No additional live qualification is needed for these bounded corrections. + +The stale pool-size example and four workers with missing nvm remain visible +observations for human spec review and operations; this contribution does not +repair or resize the shared pool. The zero-renewal-loss gate remains one observation, +not a long-term reliability rate. Pre-merge docs prep will retain the proposal and +plan as coherent design records, delete completed checkpoint/review records, and +carry essential validation facts and limitations into the PR description. diff --git a/proofs/arch/test-runner/sharded-execution.md b/proofs/arch/test-runner/sharded-execution.md index a69970fc..2e118ab8 100644 --- a/proofs/arch/test-runner/sharded-execution.md +++ b/proofs/arch/test-runner/sharded-execution.md @@ -8,7 +8,7 @@ Proves [specs/arch/test-runner/sharded-execution.md](../../../specs/arch/test-ru Coverage declarations are carried inside the migrated assertion blocks below. -The [Blaxel guard seam](blaxel-testshards.md#^blaxel-t-worker-guard) and [live temporary-directory seam](blaxel-testshards.md#^blaxel-t-tmpdir-live) prove that the outer shard attempt's ownership and `TMPDIR` survive the shared worker and nested surface environments required by [Shared shard worker](../../../specs/arch/test-runner/sharded-execution.md#Shared shard worker). The [planned live acceptance](blaxel-testshards.md#^blaxel-t-retrieval) also supplies real plan transfer. Diagnostic artifact fields and attempt-history retention in [the provider model](../../../specs/arch/test-runner/sharded-execution.md#Blaxel provider report shape) compose with [reporting's link contract](reporting.md#^reporting-t-resource-links) and [live retrieval](reporting.md#^reporting-t-resource-live). Those assertions remain pending; this proof orders no duplicate provider runs. +The [Blaxel guard seam](blaxel-testshards.md#^blaxel-t-worker-guard) and [live temporary-directory seam](blaxel-testshards.md#^blaxel-t-tmpdir-live) prove that the outer shard attempt's ownership and `TMPDIR` survive the shared worker and nested surface environments required by [Shared shard worker](../../../specs/arch/test-runner/sharded-execution.md#Shared shard worker). The [planned live acceptance](blaxel-testshards.md#^blaxel-t-retrieval) also supplies real plan transfer. Diagnostic artifact fields and attempt-history retention in [the provider model](../../../specs/arch/test-runner/sharded-execution.md#Blaxel provider report shape) compose with [reporting's link contract](reporting.md#^reporting-t-resource-links) and [live retrieval](reporting.md#^reporting-t-resource-live). Those assertions have passed; this proof relies on their retained evidence without duplicate provider runs. ## Test hooks diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index d51aaea0..4f1efca2 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -1098,6 +1098,8 @@ async function stopAndReleaseLease(lease, { killProcess }) { await lease.dispatchRequest?.catch(() => {}); // Even a successful named process wait is not a descendant audit. await confirmTeardown(lease); + // Teardown always stops owned processes. On interruption, also wait for + // the in-flight work promise to finish retaining its reports. if (killProcess && lease.workPromise) await lease.workPromise; if (lease.released) return; if (!lease.preparationRequested && !lease.executionRequested) lease.retrieved = true; diff --git a/scripts/test/blaxel-worker-lease.d.mts b/scripts/test/blaxel-worker-lease.d.mts index a9b6cf70..77f71222 100644 --- a/scripts/test/blaxel-worker-lease.d.mts +++ b/scripts/test/blaxel-worker-lease.d.mts @@ -1,6 +1,4 @@ import type { Stats } from "node:fs"; -export const DEFAULT_RECOVERY_SECONDS: number; -export const SHARD_OWNER_ENV: string; export interface LeaseInput { lockId: string; runId?: string; shardIndex?: number; shardTotal?: number; attempt?: number; attemptId?: string; owner?: string; ttlSeconds?: number; expiresAt?: number; tmpRoot?: string; resultsRoot?: string; scratch?: string; auditTokens?: string[]; audit?: boolean; operation?: string; processName?: string; pid?: number; pidFromEnv?: boolean; retrieved?: boolean; worker?: string; summaryIdentity?: Record } export interface CleanupReport { ok: boolean; before: {availableBytes: number; freeBytes: number}; after: {availableBytes: number; freeBytes: number}; removed: Record; skips: unknown[]; errors: unknown[] } export interface LeaseRecord { lockId: string; runId: string; attemptId: string; recoverAfter: number; lastRenewal: number; expiresAt: number; recovery: {discardedReports: boolean} | null } diff --git a/scripts/test/blaxel-worker-lease.mjs b/scripts/test/blaxel-worker-lease.mjs index b591cffe..1b2bcd3c 100644 --- a/scripts/test/blaxel-worker-lease.mjs +++ b/scripts/test/blaxel-worker-lease.mjs @@ -2,9 +2,6 @@ import { createHash } from "node:crypto"; import { createRequire } from "node:module"; const require = createRequire(import.meta.url); -export const DEFAULT_RECOVERY_SECONDS = 1200; -export const SHARD_OWNER_ENV = "TV_TEST_SHARD_OWNER"; - export function shardScratchPath({ tmpRoot = "/tmp", lockId, shardIndex, attempt = 1 }) { const digest = createHash("sha256").update(lockId).digest("hex").slice(0, 20); return `${tmpRoot}/tv-testshards/r-${digest}-s${shardIndex}-a${attempt}`; From ca2f902435ff7c787a39cef474193cc3ebbc51ce Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:52:09 +0000 Subject: [PATCH 57/64] Prepare harness contribution documents for review Archive the proposal and plan because they preserve the design rationale and the coherent validation structure. Delete completed investigation notebooks, review findings and checkpoint records under the docs policy; retained run artifacts and the PR description carry validation evidence. --- .../test-harness-resource-cleanup-plan.md | 2 + .../test-harness-resource-cleanup-proposal.md | 2 + .../filesystem-inspection.txt | 32 - .../gc-renderers.json | 578 ---- .../measurements.json | 2585 ----------------- .../run3-kernel-oom.txt | 8 - .../run4-kernel-oom.txt | 11 - .../chromium-insufficient-resources.md | 107 - .../tv-testshard-x64-poc-03-sizes.json | 35 - .../tv-testshard-x64-poc-03.json | 420 --- .../tv-testshard-x64-poc-131-sizes.json | 39 - .../tv-testshard-x64-poc-131.json | 502 ---- ...est-harness-resource-cleanup-integrated.md | 169 -- ...harness-resource-cleanup-plan-review-r1.md | 61 - ...harness-resource-cleanup-plan-review-r2.md | 55 - ...arness-resource-cleanup-proof-review-r1.md | 73 - ...arness-resource-cleanup-proof-review-r2.md | 61 - ...test-harness-resource-cleanup-review-r1.md | 131 - ...test-harness-resource-cleanup-review-r2.md | 72 - ...test-harness-resource-cleanup-review-r3.md | 70 - ...arness-resource-cleanup-review-response.md | 356 --- ...ness-resource-cleanup-slice-1-review-r1.md | 57 - .../test-harness-resource-cleanup-slice-1.md | 66 - ...ness-resource-cleanup-slice-2-review-r1.md | 57 - .../test-harness-resource-cleanup-slice-2.md | 68 - ...ness-resource-cleanup-slice-3-review-r1.md | 70 - ...ness-resource-cleanup-slice-3-review-r2.md | 62 - .../test-harness-resource-cleanup-slice-3.md | 398 --- ...harness-resource-cleanup-spec-review-r1.md | 65 - 29 files changed, 4 insertions(+), 6208 deletions(-) rename docs/{working => archive/2026-10}/test-harness-resource-cleanup-plan.md (98%) rename docs/{working => archive/2026-10}/test-harness-resource-cleanup-proposal.md (99%) delete mode 100644 docs/working/chromium-insufficient-resources-evidence/filesystem-inspection.txt delete mode 100644 docs/working/chromium-insufficient-resources-evidence/gc-renderers.json delete mode 100644 docs/working/chromium-insufficient-resources-evidence/measurements.json delete mode 100644 docs/working/chromium-insufficient-resources-evidence/run3-kernel-oom.txt delete mode 100644 docs/working/chromium-insufficient-resources-evidence/run4-kernel-oom.txt delete mode 100644 docs/working/chromium-insufficient-resources.md delete mode 100644 docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03-sizes.json delete mode 100644 docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03.json delete mode 100644 docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131-sizes.json delete mode 100644 docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131.json delete mode 100644 docs/working/test-harness-resource-cleanup-integrated.md delete mode 100644 docs/working/test-harness-resource-cleanup-plan-review-r1.md delete mode 100644 docs/working/test-harness-resource-cleanup-plan-review-r2.md delete mode 100644 docs/working/test-harness-resource-cleanup-proof-review-r1.md delete mode 100644 docs/working/test-harness-resource-cleanup-proof-review-r2.md delete mode 100644 docs/working/test-harness-resource-cleanup-review-r1.md delete mode 100644 docs/working/test-harness-resource-cleanup-review-r2.md delete mode 100644 docs/working/test-harness-resource-cleanup-review-r3.md delete mode 100644 docs/working/test-harness-resource-cleanup-review-response.md delete mode 100644 docs/working/test-harness-resource-cleanup-slice-1-review-r1.md delete mode 100644 docs/working/test-harness-resource-cleanup-slice-1.md delete mode 100644 docs/working/test-harness-resource-cleanup-slice-2-review-r1.md delete mode 100644 docs/working/test-harness-resource-cleanup-slice-2.md delete mode 100644 docs/working/test-harness-resource-cleanup-slice-3-review-r1.md delete mode 100644 docs/working/test-harness-resource-cleanup-slice-3-review-r2.md delete mode 100644 docs/working/test-harness-resource-cleanup-slice-3.md delete mode 100644 docs/working/test-harness-resource-cleanup-spec-review-r1.md diff --git a/docs/working/test-harness-resource-cleanup-plan.md b/docs/archive/2026-10/test-harness-resource-cleanup-plan.md similarity index 98% rename from docs/working/test-harness-resource-cleanup-plan.md rename to docs/archive/2026-10/test-harness-resource-cleanup-plan.md index afead883..e73c6596 100644 --- a/docs/working/test-harness-resource-cleanup-plan.md +++ b/docs/archive/2026-10/test-harness-resource-cleanup-plan.md @@ -1,3 +1,5 @@ +> **Archived 2026-10 from PR #12.** This plan's three slices and private qualification followed by standard shared-pool validation were completed. It preserves one coherent view of the derivation boundaries and why the two-worker full gate was rejected in favor of the standard partition. The body below is unchanged from its working state and is a clue to the change, not a record of it. + # Test-harness resource cleanup: implementation plan Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; [slice 1 has converged](test-harness-resource-cleanup-slice-1-review-r1.md), and [slice 2 has converged](test-harness-resource-cleanup-slice-2-review-r1.md). [Slice 3 has converged](test-harness-resource-cleanup-slice-3-review-r2.md) at `dc1d9f2e`. [Integrated validation](test-harness-resource-cleanup-integrated.md) passed; independent integrated review is next. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. diff --git a/docs/working/test-harness-resource-cleanup-proposal.md b/docs/archive/2026-10/test-harness-resource-cleanup-proposal.md similarity index 99% rename from docs/working/test-harness-resource-cleanup-proposal.md rename to docs/archive/2026-10/test-harness-resource-cleanup-proposal.md index 5370de28..03a852a5 100644 --- a/docs/working/test-harness-resource-cleanup-proposal.md +++ b/docs/archive/2026-10/test-harness-resource-cleanup-proposal.md @@ -1,3 +1,5 @@ +> **Archived 2026-10 from PR #12.** This proposal shaped the implemented test-harness cleanup and recovery requirements. It preserves the resource investigation's quantitative context, the reasons for limiting built-application conversion to five cases, and the tradeoffs behind one renewable recovery deadline. The body below is unchanged from its working state and is a clue to the change, not a record of it. + # Test-harness resource cleanup Converged proposal, revised 2026-10-27 with bounded proposal/spec review refinements. Owner: rose. Branch: `thopter/test-harness-resource-cleanup`, draft PR #12, originally based on `b5477211`. Main at `a8e73fbf` was integrated in `74d22941` before spec derivation. The supervisor arranges independent review of the proposal, specs, proofs, and implementation; Josh reviews every spec delta at PR time. No production behavior changes are proposed. diff --git a/docs/working/chromium-insufficient-resources-evidence/filesystem-inspection.txt b/docs/working/chromium-insufficient-resources-evidence/filesystem-inspection.txt deleted file mode 100644 index bb656303..00000000 --- a/docs/working/chromium-insufficient-resources-evidence/filesystem-inspection.txt +++ /dev/null @@ -1,32 +0,0 @@ -Read-only inspection of tv-testshard-x64-poc-131, 2026-10-01. -This is later state, not a snapshot at the original failed run's instant. - -find /tmp -maxdepth 1 -type d -name 'television-source-cli-e2e-*' | wc -l -267 - -du -h -d 1 /tmp /workspace/television /cache /home/playwright/.cache /home/playwright/.nvm -Selected output: -13M /tmp/node-compile-cache -17M /tmp/playwright-transform-cache-0 -1.2G /tmp -167M /cache/npm -241M /cache/electron -407M /cache -243M /workspace/television/.git -1.1G /workspace/television/node_modules -1.4G /workspace/television -234M /home/playwright/.nvm -915M /home/playwright/.cache/ms-playwright -1.1G /home/playwright/.cache - -Source CLI runtime directories in the largest listed entries were approximately -4.1M each. getSourceCLIEntry() has no cleanup of their shared build/runtime tree. - -df -h /tmp /dev/shm -Filesystem Size Used Avail Use% Mounted on -overlay 6.3G 4.3G 2.0G 68% / -tmpfs 3.9G 512K 3.9G 1% /dev/shm - -/proc/mounts, relevant rows: -overlay / overlay rw,relatime,lowerdir=/mnt/erofs,upperdir=/mnt/tmp/upper,workdir=/mnt/tmp/work,uuid=on 0 0 -tmpfs /dev/shm tmpfs rw,nosuid,noexec,relatime 0 0 diff --git a/docs/working/chromium-insufficient-resources-evidence/gc-renderers.json b/docs/working/chromium-insufficient-resources-evidence/gc-renderers.json deleted file mode 100644 index 5eb571b2..00000000 --- a/docs/working/chromium-insufficient-resources-evidence/gc-renderers.json +++ /dev/null @@ -1,578 +0,0 @@ -[ - { - "kind": "ROSE_GC_BEFORE", - "data": { - "at": "2026-10-01T22:21:26.951Z", - "processes": [ - { - "pid": 11694, - "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1519017284 kB", - "VmRSS:\t 94780 kB", - "Threads:\t15" - ], - "fdCount": 259, - "limits": [ - "Max processes 31819 31819 processes ", - "Max open files 4096 4096 files ", - "Max address space unlimited unlimited bytes " - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "shared-memory": 229, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4 - }, - "sharedMemorySizes": { - "131000": 8, - "2097152": 220, - "4198400": 1 - }, - "sharedMemoryAllocated": 466620416 - }, - { - "pid": 11730, - "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1519015940 kB", - "VmRSS:\t 86480 kB", - "Threads:\t15" - ], - "fdCount": 259, - "limits": [ - "Max processes 31819 31819 processes ", - "Max open files 4096 4096 files ", - "Max address space unlimited unlimited bytes " - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "shared-memory": 229, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4 - }, - "sharedMemorySizes": { - "131000": 8, - "2097152": 220, - "4198400": 1 - }, - "sharedMemoryAllocated": 466620416 - } - ], - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2730892 kB", - "MemAvailable: 2724816 kB", - "SwapFree: 0 kB", - "Shmem: 3822512 kB" - ], - "fileNr": "832\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 2762153984 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - } - } - }, - { - "kind": "ROSE_GC_AFTER", - "data": { - "at": "2026-10-01T22:21:27.072Z", - "processes": [ - { - "pid": 11694, - "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1518566212 kB", - "VmRSS:\t 55728 kB", - "Threads:\t15" - ], - "fdCount": 35, - "limits": [ - "Max processes 31819 31819 processes ", - "Max open files 4096 4096 files ", - "Max address space unlimited unlimited bytes " - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4, - "shared-memory": 5 - }, - "sharedMemorySizes": { - "131000": 4, - "4198400": 1 - }, - "sharedMemoryAllocated": 4722688 - }, - { - "pid": 11730, - "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1518564868 kB", - "VmRSS:\t 52288 kB", - "Threads:\t15" - ], - "fdCount": 35, - "limits": [ - "Max processes 31819 31819 processes ", - "Max open files 4096 4096 files ", - "Max address space unlimited unlimited bytes " - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4, - "shared-memory": 5 - }, - "sharedMemorySizes": { - "131000": 4, - "4198400": 1 - }, - "sharedMemoryAllocated": 4722688 - } - ], - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3699996 kB", - "MemAvailable: 3692256 kB", - "SwapFree: 0 kB", - "Shmem: 2920608 kB" - ], - "fileNr": "416\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 3685933056 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - } - } - }, - { - "kind": "ROSE_GC_BEFORE", - "data": { - "at": "2026-10-01T22:21:28.387Z", - "processes": [ - { - "pid": 11694, - "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1518575396 kB", - "VmRSS:\t 86100 kB", - "Threads:\t15" - ], - "fdCount": 43, - "limits": [ - "Max processes 31819 31819 processes ", - "Max open files 4096 4096 files ", - "Max address space unlimited unlimited bytes " - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4, - "shared-memory": 13 - }, - "sharedMemorySizes": { - "131000": 8, - "2097152": 4, - "4198400": 1 - }, - "sharedMemoryAllocated": 13635584 - }, - { - "pid": 11730, - "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1518573572 kB", - "VmRSS:\t 80040 kB", - "Threads:\t15" - ], - "fdCount": 43, - "limits": [ - "Max processes 31819 31819 processes ", - "Max open files 4096 4096 files ", - "Max address space unlimited unlimited bytes " - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4, - "shared-memory": 13 - }, - "sharedMemorySizes": { - "131000": 8, - "2097152": 4, - "4198400": 1 - }, - "sharedMemoryAllocated": 13635584 - } - ], - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3506724 kB", - "MemAvailable: 3498712 kB", - "SwapFree: 0 kB", - "Shmem: 2944532 kB" - ], - "fileNr": "448\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 3661377536 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - } - } - }, - { - "kind": "ROSE_GC_AFTER", - "data": { - "at": "2026-10-01T22:21:28.498Z", - "processes": [ - { - "pid": 11694, - "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1518566692 kB", - "VmRSS:\t 58788 kB", - "Threads:\t15" - ], - "fdCount": 35, - "limits": [ - "Max processes 31819 31819 processes ", - "Max open files 4096 4096 files ", - "Max address space unlimited unlimited bytes " - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4, - "shared-memory": 5 - }, - "sharedMemorySizes": { - "131000": 4, - "4198400": 1 - }, - "sharedMemoryAllocated": 4722688 - }, - { - "pid": 11730, - "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1518564868 kB", - "VmRSS:\t 54528 kB", - "Threads:\t15" - ], - "fdCount": 35, - "limits": [ - "Max processes 31819 31819 processes ", - "Max open files 4096 4096 files ", - "Max address space unlimited unlimited bytes " - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4, - "shared-memory": 5 - }, - "sharedMemorySizes": { - "131000": 4, - "4198400": 1 - }, - "sharedMemoryAllocated": 4722688 - } - ], - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3542992 kB", - "MemAvailable: 3534980 kB", - "SwapFree: 0 kB", - "Shmem: 2927288 kB" - ], - "fileNr": "416\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 3679182848 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - } - } - }, - { - "kind": "ROSE_GC_BEFORE", - "data": { - "at": "2026-10-01T22:21:29.888Z", - "processes": [ - { - "pid": 11694, - "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1518584072 kB", - "VmRSS:\t 87392 kB", - "Threads:\t16" - ], - "fdCount": 43, - "limits": [ - "Max processes 31819 31819 processes ", - "Max open files 4096 4096 files ", - "Max address space unlimited unlimited bytes " - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4, - "shared-memory": 13 - }, - "sharedMemorySizes": { - "131000": 8, - "2097152": 4, - "4198400": 1 - }, - "sharedMemoryAllocated": 13635584 - }, - { - "pid": 11730, - "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1518573572 kB", - "VmRSS:\t 81144 kB", - "Threads:\t15" - ], - "fdCount": 43, - "limits": [ - "Max processes 31819 31819 processes ", - "Max open files 4096 4096 files ", - "Max address space unlimited unlimited bytes " - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4, - "shared-memory": 13 - }, - "sharedMemorySizes": { - "131000": 8, - "2097152": 4, - "4198400": 1 - }, - "sharedMemoryAllocated": 13635584 - } - ], - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3332724 kB", - "MemAvailable: 3324684 kB", - "SwapFree: 0 kB", - "Shmem: 2951136 kB" - ], - "fileNr": "448\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 3654643712 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - } - } - }, - { - "kind": "ROSE_GC_AFTER", - "data": { - "at": "2026-10-01T22:21:29.953Z", - "processes": [ - { - "pid": 11694, - "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1518575368 kB", - "VmRSS:\t 59208 kB", - "Threads:\t16" - ], - "fdCount": 35, - "limits": [ - "Max processes 31819 31819 processes ", - "Max open files 4096 4096 files ", - "Max address space unlimited unlimited bytes " - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4, - "shared-memory": 5 - }, - "sharedMemorySizes": { - "131000": 4, - "4198400": 1 - }, - "sharedMemoryAllocated": 4722688 - }, - { - "pid": 11730, - "command": "/home/playwright/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell --type=renderer --headless=old --no-sandbox --disable-dev-shm-usage --disable-back-forward-cache --disable-background-timer-throttling --disable-breakpad --enable-automation --force-color-profile=srgb --remote-debugging-pipe --allo", - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1518564868 kB", - "VmRSS:\t 55196 kB", - "Threads:\t15" - ], - "fdCount": 35, - "limits": [ - "Max processes 31819 31819 processes ", - "Max open files 4096 4096 files ", - "Max address space unlimited unlimited bytes " - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4, - "shared-memory": 5 - }, - "sharedMemorySizes": { - "131000": 4, - "4198400": 1 - }, - "sharedMemoryAllocated": 4722688 - } - ], - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3341408 kB", - "MemAvailable: 3333368 kB", - "SwapFree: 0 kB", - "Shmem: 2933852 kB" - ], - "fileNr": "448\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 3672453120 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - } - } - } -] diff --git a/docs/working/chromium-insufficient-resources-evidence/measurements.json b/docs/working/chromium-insufficient-resources-evidence/measurements.json deleted file mode 100644 index 632844d3..00000000 --- a/docs/working/chromium-insufficient-resources-evidence/measurements.json +++ /dev/null @@ -1,2585 +0,0 @@ -[ - { - "experiment": 1, - "runDirectory": "/home/user/workspace/wt/television/investigate-chromium-insufficient-resources/.test-runs/2026-10-01T22-06-53-142Z-p2875221-r19ba7810391cbf6c", - "commit": "0ba2dcdaed7c9a2d7692be012255d4b5ceb7d5af", - "sandbox": "tv-testshard-x64-poc-143", - "result": { - "id": "2026-10-01T22-06-53-142Z-p2875221-r19ba7810391cbf6c", - "commandProvider": "blaxel", - "timingProvider": "blaxel-playwright-x64-4vcpu", - "status": "passed", - "infraStatus": "completed", - "testStatus": "passed", - "startedAt": "2026-10-01T22:06:54.717Z", - "completedAt": "2026-10-01T22:07:40.426Z", - "durationMs": 45709 - }, - "attempts": [ - { - "title": "executable theme reset reloads connected Chromium documents under confirmed state", - "retry": 0, - "status": "passed", - "durationMs": 5959, - "startTime": "2026-10-01T22:07:31.091Z", - "error": null, - "measurements": [ - { - "kind": "start", - "data": { - "at": "2026-10-01T22:07:31.311Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3734932 kB", - "MemAvailable: 3745952 kB", - "SwapFree: 0 kB", - "Shmem: 3079096 kB" - ], - "fileNr": "224\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 7478, - "kind": "network", - "fdCount": 20, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7458", - "FDSize:\t128", - "VmSize:\t50617288 kB", - "VmRSS:\t 8840 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 4, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1 - } - } - ] - } - }, - { - "kind": "peak", - "data": { - "maximumFD": 707, - "maximumPending": 261, - "failures": 0, - "sample": { - "at": "2026-10-01T22:07:37.108Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 528056 kB", - "MemAvailable: 549428 kB", - "SwapFree: 0 kB", - "Shmem: 5840544 kB" - ], - "fileNr": "1696\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 7478, - "kind": "network", - "fdCount": 42, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7458", - "FDSize:\t128", - "VmSize:\t50618576 kB", - "VmRSS:\t 20044 kB", - "Threads:\t8" - ] - }, - { - "pid": 7502, - "kind": "renderer", - "fdCount": 707, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7461", - "FDSize:\t1024", - "VmSize:\t1519911712 kB", - "VmRSS:\t 140560 kB", - "Threads:\t14" - ] - }, - { - "pid": 7540, - "kind": "renderer", - "fdCount": 707, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7461", - "FDSize:\t1024", - "VmSize:\t1519909888 kB", - "VmRSS:\t 127952 kB", - "Threads:\t14" - ] - } - ] - } - } - }, - { - "kind": "end", - "data": { - "at": "2026-10-01T22:07:37.233Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 801948 kB", - "MemAvailable: 823440 kB", - "SwapFree: 0 kB", - "Shmem: 5635056 kB" - ], - "fileNr": "992\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 7478, - "kind": "network", - "fdCount": 28, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7458", - "FDSize:\t128", - "VmSize:\t50617420 kB", - "VmRSS:\t 19576 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 11, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 1 - } - }, - { - "pid": 7502, - "kind": "renderer", - "fdCount": 707, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7461", - "FDSize:\t1024", - "VmSize:\t1519912048 kB", - "VmRSS:\t 141920 kB", - "Threads:\t14" - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "shared-memory": 677, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4 - } - } - ] - } - } - ], - "failedRequestCount": 0, - "firstFailedRequests": [] - } - ] - }, - { - "experiment": 2, - "runDirectory": "/home/user/workspace/wt/television/investigate-chromium-insufficient-resources/.test-runs/2026-10-01T22-08-43-298Z-p2875805-r39e7ed872b6bcb21", - "commit": "0ba2dcdaed7c9a2d7692be012255d4b5ceb7d5af", - "sandbox": "tv-testshard-x64-poc-39", - "result": { - "id": "2026-10-01T22-08-43-298Z-p2875805-r39e7ed872b6bcb21", - "commandProvider": "blaxel", - "timingProvider": "blaxel-playwright-x64-4vcpu", - "status": "passed", - "infraStatus": "completed", - "testStatus": "passed", - "startedAt": "2026-10-01T22:08:44.865Z", - "completedAt": "2026-10-01T22:09:29.186Z", - "durationMs": 44321 - }, - "attempts": [ - { - "title": "executable theme reset reloads connected Chromium documents under confirmed state", - "retry": 0, - "status": "passed", - "durationMs": 4359, - "startTime": "2026-10-01T22:09:22.804Z", - "error": null, - "measurements": [ - { - "kind": "start", - "data": { - "at": "2026-10-01T22:09:22.818Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2451848 kB", - "MemAvailable: 2480472 kB", - "SwapFree: 0 kB", - "Shmem: 4021684 kB" - ], - "fileNr": "288\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 8513, - "kind": "network", - "fdCount": 20, - "status": [ - "Name:\tchrome-headless", - "PPid:\t8492", - "FDSize:\t128", - "VmSize:\t50617288 kB", - "VmRSS:\t 14788 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 4, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1 - } - } - ] - } - }, - { - "kind": "peak", - "data": { - "maximumFD": 707, - "maximumPending": 265, - "failures": 0, - "sample": { - "at": "2026-10-01T22:09:27.077Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 119560 kB", - "MemAvailable: 153952 kB", - "SwapFree: 0 kB", - "Shmem: 6173108 kB" - ], - "fileNr": "1696\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 8513, - "kind": "network", - "fdCount": 42, - "status": [ - "Name:\tchrome-headless", - "PPid:\t8492", - "FDSize:\t128", - "VmSize:\t50618576 kB", - "VmRSS:\t 21588 kB", - "Threads:\t8" - ] - }, - { - "pid": 8727, - "kind": "renderer", - "fdCount": 707, - "status": [ - "Name:\tchrome-headless", - "PPid:\t8495", - "FDSize:\t1024", - "VmSize:\t1519912192 kB", - "VmRSS:\t 141560 kB", - "Threads:\t14" - ] - }, - { - "pid": 8754, - "kind": "renderer", - "fdCount": 707, - "status": [ - "Name:\tchrome-headless", - "PPid:\t8495", - "FDSize:\t1024", - "VmSize:\t1519959064 kB", - "VmRSS:\t 128964 kB", - "Threads:\t20" - ] - } - ] - } - } - }, - { - "kind": "end", - "data": { - "at": "2026-10-01T22:09:27.160Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 341484 kB", - "MemAvailable: 372820 kB", - "SwapFree: 0 kB", - "Shmem: 6159348 kB" - ], - "fileNr": "1024\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 8513, - "kind": "network", - "fdCount": 30, - "status": [ - "Name:\tchrome-headless", - "PPid:\t8492", - "FDSize:\t128", - "VmSize:\t50617552 kB", - "VmRSS:\t 21476 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 12, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 2 - } - }, - { - "pid": 8727, - "kind": "renderer", - "fdCount": 707, - "status": [ - "Name:\tchrome-headless", - "PPid:\t8495", - "FDSize:\t1024", - "VmSize:\t1519912528 kB", - "VmRSS:\t 142856 kB", - "Threads:\t14" - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "shared-memory": 677, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4 - } - } - ] - } - } - ], - "failedRequestCount": 0, - "firstFailedRequests": [] - } - ] - }, - { - "experiment": 3, - "runDirectory": "/home/user/workspace/wt/television/investigate-chromium-insufficient-resources/.test-runs/2026-10-01T22-10-17-150Z-p2876640-r6c20ebd19d66a71a", - "commit": "0ba2dcdaed7c9a2d7692be012255d4b5ceb7d5af", - "sandbox": "tv-testshard-x64-poc-15", - "result": { - "id": "2026-10-01T22-10-17-150Z-p2876640-r6c20ebd19d66a71a", - "commandProvider": "blaxel", - "timingProvider": "blaxel-playwright-x64-4vcpu", - "status": "failed", - "infraStatus": "completed", - "testStatus": "failed", - "startedAt": "2026-10-01T22:10:18.844Z", - "completedAt": "2026-10-01T22:15:08.685Z", - "durationMs": 289841 - }, - "attempts": [ - { - "title": "executable theme reset reloads connected Chromium documents under confirmed state", - "retry": 0, - "status": "timedOut", - "durationMs": 90053, - "startTime": "2026-10-01T22:11:33.511Z", - "error": "\u001b[31mTest timeout of 90000ms exceeded.\u001b[39m", - "measurements": [ - { - "kind": "start", - "data": { - "at": "2026-10-01T22:11:33.527Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 1639032 kB", - "MemAvailable: 1709856 kB", - "SwapFree: 0 kB", - "Shmem: 4582788 kB" - ], - "fileNr": "288\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 6507, - "kind": "network", - "fdCount": 20, - "status": [ - "Name:\tchrome-headless", - "PPid:\t6487", - "FDSize:\t128", - "VmSize:\t50617288 kB", - "VmRSS:\t 15032 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 4, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1 - } - } - ] - } - }, - { - "kind": "peak", - "data": { - "maximumFD": 707, - "maximumPending": 264, - "failures": 0, - "sample": { - "at": "2026-10-01T22:11:38.290Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 920584 kB", - "MemAvailable: 862864 kB", - "SwapFree: 0 kB", - "Shmem: 5410524 kB" - ], - "fileNr": "1024\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 6507, - "kind": "network", - "fdCount": 34, - "status": [ - "Name:\tchrome-headless", - "PPid:\t6487", - "FDSize:\t128", - "VmSize:\t50617932 kB", - "VmRSS:\t 20804 kB", - "Threads:\t8" - ] - }, - { - "pid": 6742, - "kind": "renderer", - "fdCount": 707, - "status": [ - "Name:\tchrome-headless", - "PPid:\t6490", - "FDSize:\t1024", - "VmSize:\t1519918084 kB", - "VmRSS:\t 128412 kB", - "Threads:\t15" - ] - } - ] - } - } - }, - { - "kind": "end", - "data": { - "at": "2026-10-01T22:13:03.553Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2545440 kB", - "MemAvailable: 2485900 kB", - "SwapFree: 0 kB", - "Shmem: 4057056 kB" - ], - "fileNr": "320\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 6507, - "kind": "network", - "fdCount": 28, - "status": [ - "Name:\tchrome-headless", - "PPid:\t6487", - "FDSize:\t128", - "VmSize:\t50617420 kB", - "VmRSS:\t 18732 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 11, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 1 - } - } - ] - } - } - ], - "failedRequestCount": 0, - "firstFailedRequests": [] - }, - { - "title": "executable theme reset reloads connected Chromium documents under confirmed state", - "retry": 1, - "status": "timedOut", - "durationMs": 90325, - "startTime": "2026-10-01T22:13:04.817Z", - "error": "\u001b[31mTest timeout of 90000ms exceeded.\u001b[39m", - "measurements": [ - { - "kind": "start", - "data": { - "at": "2026-10-01T22:13:04.943Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2884208 kB", - "MemAvailable: 2824140 kB", - "SwapFree: 0 kB", - "Shmem: 4043640 kB" - ], - "fileNr": "224\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 6896, - "kind": "network", - "fdCount": 20, - "status": [ - "Name:\tchrome-headless", - "PPid:\t6876", - "FDSize:\t128", - "VmSize:\t50617256 kB", - "VmRSS:\t 8684 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 4, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1 - } - } - ] - } - }, - { - "kind": "peak", - "data": { - "maximumFD": 707, - "maximumPending": 264, - "failures": 0, - "sample": { - "at": "2026-10-01T22:13:10.753Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 999608 kB", - "MemAvailable: 941196 kB", - "SwapFree: 0 kB", - "Shmem: 5450248 kB" - ], - "fileNr": "1024\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 6896, - "kind": "network", - "fdCount": 34, - "status": [ - "Name:\tchrome-headless", - "PPid:\t6876", - "FDSize:\t128", - "VmSize:\t50617932 kB", - "VmRSS:\t 20588 kB", - "Threads:\t8" - ] - }, - { - "pid": 6957, - "kind": "renderer", - "fdCount": 707, - "status": [ - "Name:\tchrome-headless", - "PPid:\t6879", - "FDSize:\t1024", - "VmSize:\t1519918080 kB", - "VmRSS:\t 139016 kB", - "Threads:\t15" - ] - } - ] - } - } - }, - { - "kind": "end", - "data": { - "at": "2026-10-01T22:14:34.964Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2743300 kB", - "MemAvailable: 2683412 kB", - "SwapFree: 0 kB", - "Shmem: 4096916 kB" - ], - "fileNr": "352\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 6896, - "kind": "network", - "fdCount": 28, - "status": [ - "Name:\tchrome-headless", - "PPid:\t6876", - "FDSize:\t128", - "VmSize:\t50617420 kB", - "VmRSS:\t 18272 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 11, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 1 - } - } - ] - } - } - ], - "failedRequestCount": 0, - "firstFailedRequests": [] - }, - { - "title": "executable theme reset reloads connected Chromium documents under confirmed state", - "retry": 2, - "status": "failed", - "durationMs": 9071, - "startTime": "2026-10-01T22:14:36.535Z", - "error": "Error: page.evaluate: Error: Application root did not render an expected state; last state: missing\n at eval (eval at evaluate (:290:30), :32:14)", - "measurements": [ - { - "kind": "start", - "data": { - "at": "2026-10-01T22:14:36.653Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3072172 kB", - "MemAvailable: 3012020 kB", - "SwapFree: 0 kB", - "Shmem: 4072512 kB" - ], - "fileNr": "192\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 7105, - "kind": "network", - "fdCount": 20, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7083", - "FDSize:\t128", - "VmSize:\t50617288 kB", - "VmRSS:\t 8816 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 4, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1 - } - } - ] - } - }, - { - "kind": "failure-resources", - "data": { - "at": "2026-10-01T22:14:41.120Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 127620 kB", - "MemAvailable: 70980 kB", - "SwapFree: 0 kB", - "Shmem: 6518872 kB" - ], - "fileNr": "1536\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 7105, - "kind": "network", - "fdCount": 36, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7083", - "FDSize:\t128", - "VmSize:\t50618064 kB", - "VmRSS:\t 25240 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 14, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 6 - } - }, - { - "pid": 7128, - "kind": "renderer", - "fdCount": 694, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7086", - "FDSize:\t1024", - "VmSize:\t1519897128 kB", - "VmRSS:\t 132712 kB", - "Threads:\t15" - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "shared-memory": 664, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4 - } - }, - { - "pid": 7165, - "kind": "renderer", - "fdCount": 563, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7086", - "FDSize:\t1024", - "VmSize:\t1519618820 kB", - "VmRSS:\t 114564 kB", - "Threads:\t14" - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "shared-memory": 533, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4 - } - } - ] - } - }, - { - "kind": "peak", - "data": { - "maximumFD": 694, - "maximumPending": 226, - "failures": 136, - "sample": { - "at": "2026-10-01T22:14:41.179Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 150892 kB", - "MemAvailable: 93792 kB", - "SwapFree: 0 kB", - "Shmem: 6519052 kB" - ], - "fileNr": "1536\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 7105, - "kind": "network", - "fdCount": 37, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7083", - "FDSize:\t128", - "VmSize:\t50618064 kB", - "VmRSS:\t 24680 kB", - "Threads:\t8" - ] - }, - { - "pid": 7128, - "kind": "renderer", - "fdCount": 694, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7086", - "FDSize:\t1024", - "VmSize:\t1519897128 kB", - "VmRSS:\t 132712 kB", - "Threads:\t15" - ] - }, - { - "pid": 7165, - "kind": "renderer", - "fdCount": 563, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7086", - "FDSize:\t1024", - "VmSize:\t1519618820 kB", - "VmRSS:\t 113996 kB", - "Threads:\t14" - ] - } - ] - } - } - }, - { - "kind": "end", - "data": { - "at": "2026-10-01T22:14:45.682Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 374264 kB", - "MemAvailable: 317216 kB", - "SwapFree: 0 kB", - "Shmem: 6385496 kB" - ], - "fileNr": "992\t0\t814457", - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 7105, - "kind": "network", - "fdCount": 28, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7083", - "FDSize:\t128", - "VmSize:\t50617420 kB", - "VmRSS:\t 20116 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 11, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 1 - } - }, - { - "pid": 7128, - "kind": "renderer", - "fdCount": 694, - "status": [ - "Name:\tchrome-headless", - "PPid:\t7086", - "FDSize:\t1024", - "VmSize:\t1519889096 kB", - "VmRSS:\t 124724 kB", - "Threads:\t15" - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "shared-memory": 664, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4 - } - } - ] - } - } - ], - "failedRequestCount": 136, - "firstFailedRequests": [ - { - "url": "http://127.0.0.1:39101/packages/artifact/src/missing-artifact-page.ts", - "pending": 136, - "maximumPending": 226 - }, - { - "url": "http://127.0.0.1:39101/packages/artifact/src/link-target.ts", - "pending": 135, - "maximumPending": 226 - }, - { - "url": "http://127.0.0.1:39101/packages/artifact/src/index.ts", - "pending": 134, - "maximumPending": 226 - } - ] - } - ] - }, - { - "experiment": 4, - "runDirectory": "/home/user/workspace/wt/television/investigate-chromium-insufficient-resources/.test-runs/2026-10-01T22-14-34-254Z-p2879064-rb0056a09a8053d4b", - "commit": "8a75e3a9d2d59e3e3b8ffb6550aa05a79b8b7d2b", - "sandbox": "tv-testshard-x64-poc-32", - "result": { - "id": "2026-10-01T22-14-34-254Z-p2879064-rb0056a09a8053d4b", - "commandProvider": "blaxel", - "timingProvider": "blaxel-playwright-x64-4vcpu", - "status": "failed", - "infraStatus": "completed", - "testStatus": "failed", - "startedAt": "2026-10-01T22:14:35.874Z", - "completedAt": "2026-10-01T22:19:49.221Z", - "durationMs": 313347 - }, - "attempts": [ - { - "title": "executable theme reset reloads connected Chromium documents under confirmed state", - "retry": 0, - "status": "timedOut", - "durationMs": 90073, - "startTime": "2026-10-01T22:15:14.179Z", - "error": "\u001b[31mTest timeout of 90000ms exceeded.\u001b[39m", - "measurements": [ - { - "kind": "start", - "data": { - "at": "2026-10-01T22:15:14.513Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2503260 kB", - "MemAvailable: 2577740 kB", - "SwapFree: 0 kB", - "Shmem: 4181444 kB" - ], - "fileNr": "192\t0\t814457", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 2394783744 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172300288 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 1826, - "kind": "network", - "fdCount": 20, - "status": [ - "Name:\tchrome-headless", - "PPid:\t1805", - "FDSize:\t128", - "VmSize:\t50617288 kB", - "VmRSS:\t 8788 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 4, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1 - }, - "sharedMemorySizes": {}, - "sharedMemoryAllocated": 0 - } - ] - } - }, - { - "kind": "peak", - "data": { - "maximumFD": 707, - "maximumPending": 259, - "failures": 0, - "sample": { - "at": "2026-10-01T22:15:20.351Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 980656 kB", - "MemAvailable: 926220 kB", - "SwapFree: 0 kB", - "Shmem: 5575128 kB" - ], - "fileNr": "992\t0\t814457", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 967827456 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172300288 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 1826, - "kind": "network", - "fdCount": 34, - "status": [ - "Name:\tchrome-headless", - "PPid:\t1805", - "FDSize:\t128", - "VmSize:\t50617932 kB", - "VmRSS:\t 18420 kB", - "Threads:\t8" - ] - }, - { - "pid": 1888, - "kind": "renderer", - "fdCount": 707, - "status": [ - "Name:\tchrome-headless", - "PPid:\t1808", - "FDSize:\t1024", - "VmSize:\t1519918084 kB", - "VmRSS:\t 128300 kB", - "Threads:\t15" - ] - } - ] - } - } - }, - { - "kind": "lowest-resources", - "data": { - "minimumDiskAvailable": 279891968, - "minimumMemoryAvailableKB": 181372, - "sample": { - "at": "2026-10-01T22:15:19.853Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 136628 kB", - "MemAvailable: 181372 kB", - "SwapFree: 0 kB", - "Shmem: 6246452 kB" - ], - "fileNr": "1344\t0\t814457", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 279891968 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172300288 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 1826, - "kind": "network", - "fdCount": 36, - "status": [ - "Name:\tchrome-headless", - "PPid:\t1805", - "FDSize:\t128", - "VmSize:\t50618064 kB", - "VmRSS:\t 24920 kB", - "Threads:\t8" - ] - }, - { - "pid": 1850, - "kind": "renderer", - "fdCount": 548, - "status": [ - "Name:\tchrome-headless", - "PPid:\t1808", - "FDSize:\t1024", - "VmSize:\t1519598596 kB", - "VmRSS:\t 134136 kB", - "Threads:\t15" - ] - }, - { - "pid": 1888, - "kind": "renderer", - "fdCount": 524, - "status": [ - "Name:\tchrome-headless", - "PPid:\t1808", - "FDSize:\t1024", - "VmSize:\t1519547140 kB", - "VmRSS:\t 121612 kB", - "Threads:\t15" - ] - } - ] - } - } - }, - { - "kind": "end", - "data": { - "at": "2026-10-01T22:16:44.525Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2484596 kB", - "MemAvailable: 2428152 kB", - "SwapFree: 0 kB", - "Shmem: 4221796 kB" - ], - "fileNr": "320\t0\t814457", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 2353655808 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172300288 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 1826, - "kind": "network", - "fdCount": 28, - "status": [ - "Name:\tchrome-headless", - "PPid:\t1805", - "FDSize:\t128", - "VmSize:\t50617420 kB", - "VmRSS:\t 16712 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 11, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 1 - }, - "sharedMemorySizes": { - "16": 1 - }, - "sharedMemoryAllocated": 4096 - } - ] - } - } - ], - "failedRequestCount": 0, - "firstFailedRequests": [] - }, - { - "title": "executable theme reset reloads connected Chromium documents under confirmed state", - "retry": 1, - "status": "timedOut", - "durationMs": 90396, - "startTime": "2026-10-01T22:16:45.952Z", - "error": "\u001b[31mTest timeout of 90000ms exceeded.\u001b[39m", - "measurements": [ - { - "kind": "start", - "data": { - "at": "2026-10-01T22:16:46.118Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2728680 kB", - "MemAvailable: 2671804 kB", - "SwapFree: 0 kB", - "Shmem: 4208548 kB" - ], - "fileNr": "256\t0\t814457", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 2367131648 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172300288 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 2047, - "kind": "network", - "fdCount": 20, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2020", - "FDSize:\t128", - "VmSize:\t50617244 kB", - "VmRSS:\t 8284 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 4, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1 - }, - "sharedMemorySizes": {}, - "sharedMemoryAllocated": 0 - } - ] - } - }, - { - "kind": "peak", - "data": { - "maximumFD": 707, - "maximumPending": 266, - "failures": 0, - "sample": { - "at": "2026-10-01T22:16:51.998Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 851524 kB", - "MemAvailable: 796264 kB", - "SwapFree: 0 kB", - "Shmem: 5615176 kB" - ], - "fileNr": "992\t0\t814457", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 926785536 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172300288 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 2047, - "kind": "network", - "fdCount": 34, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2020", - "FDSize:\t128", - "VmSize:\t50617932 kB", - "VmRSS:\t 19272 kB", - "Threads:\t8" - ] - }, - { - "pid": 2102, - "kind": "renderer", - "fdCount": 707, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2023", - "FDSize:\t1024", - "VmSize:\t1519918676 kB", - "VmRSS:\t 139908 kB", - "Threads:\t15" - ] - } - ] - } - } - }, - { - "kind": "lowest-resources", - "data": { - "minimumDiskAvailable": 267485184, - "minimumMemoryAvailableKB": 69600, - "sample": { - "at": "2026-10-01T22:16:51.751Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 123212 kB", - "MemAvailable: 69600 kB", - "SwapFree: 0 kB", - "Shmem: 6258792 kB" - ], - "fileNr": "1312\t0\t814457", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 267485184 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172300288 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 2047, - "kind": "network", - "fdCount": 36, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2020", - "FDSize:\t128", - "VmSize:\t50618064 kB", - "VmRSS:\t 24616 kB", - "Threads:\t8" - ] - }, - { - "pid": 2065, - "kind": "renderer", - "fdCount": 362, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2023", - "FDSize:\t512", - "VmSize:\t1519208992 kB", - "VmRSS:\t 132004 kB", - "Threads:\t14" - ] - }, - { - "pid": 2102, - "kind": "renderer", - "fdCount": 696, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2023", - "FDSize:\t1024", - "VmSize:\t1519899396 kB", - "VmRSS:\t 121356 kB", - "Threads:\t15" - ] - } - ] - } - } - }, - { - "kind": "end", - "data": { - "at": "2026-10-01T22:18:16.137Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2540924 kB", - "MemAvailable: 2484324 kB", - "SwapFree: 0 kB", - "Shmem: 4261040 kB" - ], - "fileNr": "352\t0\t814457", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 2313469952 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172300288 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 2047, - "kind": "network", - "fdCount": 28, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2020", - "FDSize:\t128", - "VmSize:\t50617420 kB", - "VmRSS:\t 17128 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 11, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 1 - }, - "sharedMemorySizes": { - "16": 1 - }, - "sharedMemoryAllocated": 4096 - } - ] - } - } - ], - "failedRequestCount": 0, - "firstFailedRequests": [] - }, - { - "title": "executable theme reset reloads connected Chromium documents under confirmed state", - "retry": 2, - "status": "timedOut", - "durationMs": 90068, - "startTime": "2026-10-01T22:18:17.974Z", - "error": "\u001b[31mTest timeout of 90000ms exceeded.\u001b[39m", - "measurements": [ - { - "kind": "start", - "data": { - "at": "2026-10-01T22:18:18.103Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2860164 kB", - "MemAvailable: 2803208 kB", - "SwapFree: 0 kB", - "Shmem: 4237056 kB" - ], - "fileNr": "224\t0\t814457", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 2337763328 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172300288 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 2254, - "kind": "network", - "fdCount": 19, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2232", - "FDSize:\t128", - "VmSize:\t50617244 kB", - "VmRSS:\t 8292 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 8, - "socket": 4, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1 - }, - "sharedMemorySizes": {}, - "sharedMemoryAllocated": 0 - } - ] - } - }, - { - "kind": "peak", - "data": { - "maximumFD": 707, - "maximumPending": 267, - "failures": 0, - "sample": { - "at": "2026-10-01T22:18:23.193Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 1069824 kB", - "MemAvailable: 1014404 kB", - "SwapFree: 0 kB", - "Shmem: 5631696 kB" - ], - "fileNr": "1024\t0\t814457", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 909901824 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172300288 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 2254, - "kind": "network", - "fdCount": 34, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2232", - "FDSize:\t128", - "VmSize:\t50617932 kB", - "VmRSS:\t 19736 kB", - "Threads:\t8" - ] - }, - { - "pid": 2313, - "kind": "renderer", - "fdCount": 707, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2235", - "FDSize:\t1024", - "VmSize:\t1519910148 kB", - "VmRSS:\t 127852 kB", - "Threads:\t14" - ] - } - ] - } - } - }, - { - "kind": "lowest-resources", - "data": { - "minimumDiskAvailable": 58142720, - "minimumMemoryAvailableKB": 84688, - "sample": { - "at": "2026-10-01T22:18:22.698Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 138392 kB", - "MemAvailable: 84688 kB", - "SwapFree: 0 kB", - "Shmem: 6463200 kB" - ], - "fileNr": "1440\t0\t814457", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 58142720 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172300288 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 2254, - "kind": "network", - "fdCount": 36, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2232", - "FDSize:\t128", - "VmSize:\t50618064 kB", - "VmRSS:\t 23876 kB", - "Threads:\t8" - ] - }, - { - "pid": 2277, - "kind": "renderer", - "fdCount": 606, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2235", - "FDSize:\t1024", - "VmSize:\t1519717384 kB", - "VmRSS:\t 135244 kB", - "Threads:\t15" - ] - }, - { - "pid": 2313, - "kind": "renderer", - "fdCount": 544, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2235", - "FDSize:\t1024", - "VmSize:\t1519579908 kB", - "VmRSS:\t 121728 kB", - "Threads:\t14" - ] - } - ] - } - } - }, - { - "kind": "end", - "data": { - "at": "2026-10-01T22:19:48.124Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2613900 kB", - "MemAvailable: 2557356 kB", - "SwapFree: 0 kB", - "Shmem: 4278036 kB" - ], - "fileNr": "352\t0\t814457", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 2296057856 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172300288 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 2254, - "kind": "network", - "fdCount": 28, - "status": [ - "Name:\tchrome-headless", - "PPid:\t2232", - "FDSize:\t128", - "VmSize:\t50617420 kB", - "VmRSS:\t 17824 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 11, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 1 - }, - "sharedMemorySizes": { - "16": 1 - }, - "sharedMemoryAllocated": 4096 - } - ] - } - } - ], - "failedRequestCount": 0, - "firstFailedRequests": [] - } - ] - }, - { - "experiment": 5, - "runDirectory": "/home/user/workspace/wt/television/investigate-chromium-insufficient-resources/.test-runs/2026-10-01T22-21-04-731Z-p2882965-r4564bb4b4668cd6d", - "commit": "9743dbe7b734adbe3e0a6658f8f6c7611ef49389", - "sandbox": "tv-testshard-x64-poc-03", - "result": { - "id": "2026-10-01T22-21-04-731Z-p2882965-r4564bb4b4668cd6d", - "commandProvider": "blaxel", - "timingProvider": "blaxel-playwright-x64-4vcpu", - "status": "passed", - "infraStatus": "completed", - "testStatus": "passed", - "startedAt": "2026-10-01T22:21:06.332Z", - "completedAt": "2026-10-01T22:21:34.890Z", - "durationMs": 28558 - }, - "attempts": [ - { - "title": "executable theme reset reloads connected Chromium documents under confirmed state", - "retry": 0, - "status": "passed", - "durationMs": 7021, - "startTime": "2026-10-01T22:21:22.913Z", - "error": null, - "measurements": [ - { - "kind": "start", - "data": { - "at": "2026-10-01T22:21:23.157Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3946536 kB", - "MemAvailable: 3936572 kB", - "SwapFree: 0 kB", - "Shmem: 2877360 kB" - ], - "fileNr": "224\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 3729829888 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 11666, - "kind": "network", - "fdCount": 11, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11646", - "FDSize:\t128", - "VmSize:\t50559728 kB", - "VmRSS:\t 7512 kB", - "Threads:\t1" - ], - "descriptorKinds": { - "file": 8, - "socket": 3 - }, - "sharedMemorySizes": {}, - "sharedMemoryAllocated": 0 - } - ] - } - }, - { - "kind": "ROSE_GC_BEFORE", - "data": { - "at": "2026-10-01T22:21:26.951Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2730892 kB", - "MemAvailable: 2724816 kB", - "SwapFree: 0 kB", - "Shmem: 3822512 kB" - ], - "fileNr": "832\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 2762153984 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 11666, - "kind": "network", - "fdCount": 42, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11646", - "FDSize:\t128", - "VmSize:\t50618576 kB", - "VmRSS:\t 19800 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 16, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 10 - }, - "sharedMemorySizes": { - "16": 2, - "131000": 8 - }, - "sharedMemoryAllocated": 1056768 - }, - { - "pid": 11694, - "kind": "renderer", - "fdCount": 259, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1519017284 kB", - "VmRSS:\t 94780 kB", - "Threads:\t15" - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "shared-memory": 229, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4 - }, - "sharedMemorySizes": { - "131000": 8, - "2097152": 220, - "4198400": 1 - }, - "sharedMemoryAllocated": 466620416 - }, - { - "pid": 11730, - "kind": "renderer", - "fdCount": 259, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1519015940 kB", - "VmRSS:\t 86480 kB", - "Threads:\t15" - ], - "descriptorKinds": { - "file": 11, - "socket": 11, - "shared-memory": 229, - "anon_inode:[eventfd]": 4, - "anon_inode:[eventpoll]": 4 - }, - "sharedMemorySizes": { - "131000": 8, - "2097152": 220, - "4198400": 1 - }, - "sharedMemoryAllocated": 466620416 - } - ] - } - }, - { - "kind": "ROSE_GC_AFTER", - "data": { - "at": "2026-10-01T22:21:27.072Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3699996 kB", - "MemAvailable: 3692256 kB", - "SwapFree: 0 kB", - "Shmem: 2920608 kB" - ], - "fileNr": "416\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 3685933056 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 11666, - "kind": "network", - "fdCount": 42, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11646", - "FDSize:\t128", - "VmSize:\t50618576 kB", - "VmRSS:\t 19800 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 16, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 10 - }, - "sharedMemorySizes": { - "16": 2, - "131000": 8 - }, - "sharedMemoryAllocated": 1056768 - } - ] - } - }, - { - "kind": "ROSE_GC_BEFORE", - "data": { - "at": "2026-10-01T22:21:28.387Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3506724 kB", - "MemAvailable: 3498712 kB", - "SwapFree: 0 kB", - "Shmem: 2944532 kB" - ], - "fileNr": "448\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 3661377536 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 11666, - "kind": "network", - "fdCount": 42, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11646", - "FDSize:\t128", - "VmSize:\t50618576 kB", - "VmRSS:\t 19992 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 16, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 10 - }, - "sharedMemorySizes": { - "16": 2, - "131000": 8 - }, - "sharedMemoryAllocated": 1056768 - } - ] - } - }, - { - "kind": "ROSE_GC_AFTER", - "data": { - "at": "2026-10-01T22:21:28.498Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3542992 kB", - "MemAvailable: 3534980 kB", - "SwapFree: 0 kB", - "Shmem: 2927288 kB" - ], - "fileNr": "416\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 3679182848 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 11666, - "kind": "network", - "fdCount": 42, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11646", - "FDSize:\t128", - "VmSize:\t50618576 kB", - "VmRSS:\t 19992 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 16, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 10 - }, - "sharedMemorySizes": { - "16": 2, - "131000": 8 - }, - "sharedMemoryAllocated": 1056768 - } - ] - } - }, - { - "kind": "ROSE_GC_BEFORE", - "data": { - "at": "2026-10-01T22:21:29.888Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3332724 kB", - "MemAvailable: 3324684 kB", - "SwapFree: 0 kB", - "Shmem: 2951136 kB" - ], - "fileNr": "448\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 3654643712 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 11666, - "kind": "network", - "fdCount": 42, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11646", - "FDSize:\t128", - "VmSize:\t50618576 kB", - "VmRSS:\t 20172 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 16, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 10 - }, - "sharedMemorySizes": { - "16": 2, - "131000": 8 - }, - "sharedMemoryAllocated": 1056768 - } - ] - } - }, - { - "kind": "ROSE_GC_AFTER", - "data": { - "at": "2026-10-01T22:21:29.953Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3341408 kB", - "MemAvailable: 3333368 kB", - "SwapFree: 0 kB", - "Shmem: 2933852 kB" - ], - "fileNr": "448\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 3672453120 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 11666, - "kind": "network", - "fdCount": 42, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11646", - "FDSize:\t128", - "VmSize:\t50618576 kB", - "VmRSS:\t 20172 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 16, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 10 - }, - "sharedMemorySizes": { - "16": 2, - "131000": 8 - }, - "sharedMemoryAllocated": 1056768 - } - ] - } - }, - { - "kind": "peak", - "data": { - "maximumFD": 255, - "maximumPending": 268, - "failures": 0, - "sample": { - "at": "2026-10-01T22:21:26.834Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2772672 kB", - "MemAvailable: 2766592 kB", - "SwapFree: 0 kB", - "Shmem: 3806180 kB" - ], - "fileNr": "832\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 2779287552 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 11666, - "kind": "network", - "fdCount": 42, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11646", - "FDSize:\t128", - "VmSize:\t50618576 kB", - "VmRSS:\t 19796 kB", - "Threads:\t8" - ] - }, - { - "pid": 11694, - "kind": "renderer", - "fdCount": 255, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1519009092 kB", - "VmRSS:\t 91164 kB", - "Threads:\t15" - ] - }, - { - "pid": 11730, - "kind": "renderer", - "fdCount": 255, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1519007748 kB", - "VmRSS:\t 83836 kB", - "Threads:\t15" - ] - } - ] - } - } - }, - { - "kind": "lowest-resources", - "data": { - "minimumDiskAvailable": 2779287552, - "minimumMemoryAvailableKB": 2766592, - "sample": { - "at": "2026-10-01T22:21:26.834Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 2772672 kB", - "MemAvailable: 2766592 kB", - "SwapFree: 0 kB", - "Shmem: 3806180 kB" - ], - "fileNr": "832\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 2779287552 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 11666, - "kind": "network", - "fdCount": 42, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11646", - "FDSize:\t128", - "VmSize:\t50618576 kB", - "VmRSS:\t 19796 kB", - "Threads:\t8" - ] - }, - { - "pid": 11694, - "kind": "renderer", - "fdCount": 255, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1519009092 kB", - "VmRSS:\t 91164 kB", - "Threads:\t15" - ] - }, - { - "pid": 11730, - "kind": "renderer", - "fdCount": 255, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11649", - "FDSize:\t512", - "VmSize:\t1519007748 kB", - "VmRSS:\t 83836 kB", - "Threads:\t15" - ] - } - ] - } - } - }, - { - "kind": "end", - "data": { - "at": "2026-10-01T22:21:30.129Z", - "memory": [ - "MemTotal: 8149540 kB", - "MemFree: 3415728 kB", - "MemAvailable: 3407672 kB", - "SwapFree: 0 kB", - "Shmem: 2922468 kB" - ], - "fileNr": "384\t0\t814457", - "oomKills": "oom_kill 0", - "filesystems": { - "/tmp": { - "total": 6676103168, - "available": 3683971072 - }, - "/dev/shm": { - "total": 4172562432, - "available": 4172562432 - } - }, - "cgroup": { - "memory.current": "unavailable", - "memory.max": "unavailable", - "memory.events": "unavailable", - "pids.current": "unavailable", - "pids.max": "unavailable", - "cpu.stat": "unavailable" - }, - "processes": [ - { - "pid": 11666, - "kind": "network", - "fdCount": 28, - "status": [ - "Name:\tchrome-headless", - "PPid:\t11646", - "FDSize:\t128", - "VmSize:\t50617420 kB", - "VmRSS:\t 19656 kB", - "Threads:\t8" - ], - "descriptorKinds": { - "file": 9, - "socket": 11, - "anon_inode:[eventfd]": 3, - "anon_inode:[eventpoll]": 3, - "anon_inode:inotify": 1, - "shared-memory": 1 - }, - "sharedMemorySizes": { - "16": 1 - }, - "sharedMemoryAllocated": 4096 - } - ] - } - } - ], - "failedRequestCount": 0, - "firstFailedRequests": [] - } - ] - } -] diff --git a/docs/working/chromium-insufficient-resources-evidence/run3-kernel-oom.txt b/docs/working/chromium-insufficient-resources-evidence/run3-kernel-oom.txt deleted file mode 100644 index a59317f9..00000000 --- a/docs/working/chromium-insufficient-resources-evidence/run3-kernel-oom.txt +++ /dev/null @@ -1,8 +0,0 @@ -[14494.236116] Chrome_ChildIOT invoked oom-killer: gfp_mask=0x140cca(GFP_HIGHUSER_MOVABLE|__GFP_COMP), order=0, oom_score_adj=0 -[14494.238238] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=chrome-headless,pid=6721,uid=0 -[14494.238317] Out of memory: Killed process 6721 (chrome-headless) total-vm:1519582212kB, anon-rss:123652kB, file-rss:0kB, shmem-rss:9544kB, UID:0 pgtables:1020kB oom_score_adj:300 -[14586.820194] libuv-worker invoked oom-killer: gfp_mask=0x140cca(GFP_HIGHUSER_MOVABLE|__GFP_COMP), order=0, oom_score_adj=0 -[14586.820722] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=chrome-headless,pid=6921,uid=0 -[14586.820769] Out of memory: Killed process 6921 (chrome-headless) total-vm:1519461376kB, anon-rss:122196kB, file-rss:0kB, shmem-rss:9892kB, UID:0 pgtables:1052kB oom_score_adj:300 -14741.33 45998.36 -Thu Oct 1 22:20:48 UTC 2026 diff --git a/docs/working/chromium-insufficient-resources-evidence/run4-kernel-oom.txt b/docs/working/chromium-insufficient-resources-evidence/run4-kernel-oom.txt deleted file mode 100644 index b20522bd..00000000 --- a/docs/working/chromium-insufficient-resources-evidence/run4-kernel-oom.txt +++ /dev/null @@ -1,11 +0,0 @@ -[20373.019890] Chrome_ChildIOT invoked oom-killer: gfp_mask=0x140cca(GFP_HIGHUSER_MOVABLE|__GFP_COMP), order=0, oom_score_adj=0 -[20373.021712] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=chrome-headless,pid=1850,uid=0 -[20373.021763] Out of memory: Killed process 1850 (chrome-headless) total-vm:1519621124kB, anon-rss:124324kB, file-rss:0kB, shmem-rss:9596kB, UID:0 pgtables:1040kB oom_score_adj:300 -[20464.830001] Chrome_ChildIOT invoked oom-killer: gfp_mask=0x140cc0(GFP_USER|__GFP_COMP), order=0, oom_score_adj=0 -[20464.830545] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=chrome-headless,pid=2065,uid=0 -[20464.830607] Out of memory: Killed process 2065 (chrome-headless) total-vm:1519254048kB, anon-rss:122396kB, file-rss:0kB, shmem-rss:9684kB, UID:0 pgtables:1048kB oom_score_adj:300 -[20555.798387] Chrome_ChildIOT invoked oom-killer: gfp_mask=0x140cc0(GFP_USER|__GFP_COMP), order=0, oom_score_adj=0 -[20555.798866] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=chrome-headless,pid=2277,uid=0 -[20555.798923] Out of memory: Killed process 2277 (chrome-headless) total-vm:1519766536kB, anon-rss:125460kB, file-rss:0kB, shmem-rss:9820kB, UID:0 pgtables:1016kB oom_score_adj:300 -20674.07 66771.89 -Thu Oct 1 22:20:48 UTC 2026 diff --git a/docs/working/chromium-insufficient-resources.md b/docs/working/chromium-insufficient-resources.md deleted file mode 100644 index 21898a03..00000000 --- a/docs/working/chromium-insufficient-resources.md +++ /dev/null @@ -1,107 +0,0 @@ -# Chromium resource failures on Blaxel - -Investigator: rose, 2026-10-01. Branch: `thopter/investigate-chromium-insufficient-resources`, based on `68b8ab11`. All five experiments ran through the canonical runner on committed, pushed revisions, on Blaxel, with the default two retries. No tests, browsers, application servers, Electron, builds, lint, or type checks ran on the shared development host. - -## Finding - -The resource pressure is **Chromium response-body shared memory competing with a persistent, RAM-backed sandbox filesystem**, with two observed failure modes: failed resource loads and kernel OOM kills of renderers. Neither the entire suite nor another shard is necessary to reproduce it. - -The unbundled Vite application makes hundreds of small module requests. Chromium 148 allocates 2 MiB shared-memory response buffers, many of which remain open after loading and navigation until garbage collection. The two-page theme-reset test loads each document four times. A passing run reached 707 descriptors in each renderer, predominantly shared-memory handles, and consumed several GiB beyond the sandbox's baseline. Forcing garbage collection between reloads released hundreds of these buffers and prevented that accumulation. - -Persistent Blaxel workers already use substantial RAM for their writable filesystem. Playwright launches Chromium with `--disable-dev-shm-usage`, placing shared-memory files under `/tmp` on that filesystem. The root filesystem reported a 6,676,103,168-byte capacity on the 8 GB workers; `/dev/shm` was almost empty. Persisted test runtime directories reduce the space and RAM left for browser buffers. A concrete leak is the cached CLI runtime created by `getSourceCLIEntry()` in `test/helpers/product-server.ts`: it makes a `television-source-cli-e2e-*` directory per worker and never removes it. The original failing sandbox, `poc-131`, had **267 such directories**, with approximately **1.2 GB under `/tmp`** at inspection. - -**Confidence boundary:** RAM exhaustion and renderer OOM kills are directly proven by kernel logs. The `ERR_INSUFFICIENT_RESOURCES` reproduction shows exhausted shared-memory/storage capacity and excludes socket/admission limits for an inspected failed request. The exact failing allocation syscall and errno were not captured; attributing that error specifically to the Mojo response-pipe allocation is a strong source-supported inference, not a syscall trace. - -## Reproduction results - -| Experiment | Revision | Blaxel sandbox | Selection and result | -|---|---|---|---| -| 1 | `0ba2dcda` | `poc-143` | Original theme-reset case plus observation: passed first attempt. Sampled peak: 549,428 KiB available RAM, 5,840,544 KiB shared memory, 707 maximum process descriptors, 261 maximum outstanding requests. | -| 2 | `0ba2dcda` | `poc-39` | Whole theme acceptance file: passed. Theme-reset sampled peak: only 153,952 KiB available RAM, 6,173,108 KiB shared memory, 707 descriptors, 265 outstanding requests. | -| 3 | `0ba2dcda` | `poc-15` | Only original shard 8's assignment: theme reset failed all three attempts. Attempts 0/1 timed out after renderer OOM kills; attempt 2 recorded **136 ERR_INSUFFICIENT_RESOURCES requests**. | -| 4 | `8a75e3a9` | `poc-32` | Original theme-reset case alone, with filesystem measurements: all three attempts timed out. Kernel logs identify **one killed Chromium renderer per attempt**. No request-failed error was required for this failure mode. | -| 5 | `9743dbe7` | `poc-03` | Same case, with diagnostic `page.requestGC()` after each completed reload step: passed first attempt, no OOM kills. First GC released **220 buffers of 2,097,152 bytes in each of two renderers**. Shared memory fell from 3,822,512 to 2,920,608 KiB; root space increased by 923,779,072 bytes. | - -The GC run used a different sandbox with a lower baseline; its pass alone is not a controlled estimate of flake reduction. The within-process disappearance of 440 2 MiB buffers is the causal evidence for retained response buffers. Subsequent reloads stayed near the lower shared-memory level. This is an investigation technique, not a proposed test fix. - -In experiment 3's request-failure snapshot: - -- Available RAM was 70,980 KiB and shared memory was 6,518,872 KiB. -- The two renderers had 694 and 563 descriptors, including 664 and 533 shared-memory descriptors. Every inspected process had a 4,096-descriptor limit. The network service was far below that limit. -- The kernel killed renderer PIDs 6721 and 6921 during attempts 0/1. Their PIDs match the experiment's process records. The final retry reached allocation failures instead. -- Shared memory was within about 1 MiB of the worker's root-filesystem capacity. That instrumentation revision did not record `statfs`, so this is a capacity comparison, not a sampled free-byte count at that error. -- Its NetLog contains eight malformed lines consistent with interrupted writes; do not treat the whole file as valid JSON. Parsing complete lines shows the first failed module request (`missing-artifact-page.ts`, NetLog source 5561) obtained an **HTTP 304 response**, reached `URL_REQUEST_DELEGATE_RESPONSE_STARTED`, and was then cancelled. Its request was admitted and its socket transaction completed. The later failure is not a socket-pool or outstanding-request admission rejection. - -Experiment 4 sampled available RAM down to 69,600 KiB. Kernel logs identify renderer PIDs 1850, 2065, and 2277, corresponding to its three attempts. The minimum sampled `/tmp` free space was 279,891,968, 267,485,184, and 58,142,720 bytes by attempt. Physical memory can exhaust before the filesystem quota; a 250 ms sampler also cannot establish the instantaneous minimum. The absence of ERR_INSUFFICIENT_RESOURCES does not indicate a healthy browser. - -## Original run and shard packing - -Original run: - -`/home/user/workspace/wt/television-desktop-connect-links-plan/.test-runs/2026-10-01T21-41-23-867Z-p2861143-ra3e7bfaa4a5003cd` - -The supplied retry-1 trace, `/tmp/desktop-connect-theme-reset-blaxel-trace.zip`, records Chromium `148.0.7778.96`, 2,017 requests across two pages, four document loads per page, and 16 consecutive SVG-module failures during the fourth load on one page. Each page has approximately 258 unique URLs; total requests must not be mistaken for simultaneous requests. - -Shard 8 runs six unit files, `token-only-persistence.test.ts`, `theme-product-acceptance.test.ts`, then desktop `user-data-identity.test.ts`. Surfaces execute sequentially; theme acceptance uses one Playwright worker. No process leak was reported. Experiment 3 reproduced its failure without the other 35 shards; experiment 4 reproduced OOM in the single case without earlier files. Simultaneous shards, a particular preceding test, and retained processes from earlier files are **not necessary causes**. - -The full-verify correlation is better explained by available capacity on the assigned persistent worker than by a unique full-verify execution mode. Baseline shared memory varied substantially: approximately 2.9 GB on experiment 5's worker versus 4.2 GB on experiment 4's. The original failing worker had only about 2 GB root space free at later inspection. That persisted filesystem is a contributor, not a measurement of exact free space at the historical failure instant. - -## Responsible harness and test behavior - -1. `test/helpers/product-server.ts:getSourceCLIEntry()` creates a temporary build/runtime tree and retains its filename in module state. `disposeAllProductServers()` removes server homes and optional bundled runtime copies, but not the shared source-CLI tree. There is no worker-exit cleanup for it. Blaxel checkout reset cleans the repository and `.testshards` results, not these `/tmp` directories. Worker reuse accumulates hundreds of them. -2. `ProductServer.appURL()` creates a development proxy to Vite. Tests navigating to `/packages/web/src/index.html` load the source-module graph. Theme reset repeatedly reloads two documents together, making buffer accumulation unusually large. The preceding executable-theme acceptance also uses two development pages, but its buffers need not survive into the reset test to cause the failure. -3. Playwright's default `--disable-dev-shm-usage` couples response buffers to the writable root filesystem. The mount reports `upperdir=/mnt/tmp/upper`; writable files contribute to shared-memory pressure. `/dev/shm` stays effectively unused. Moving allocations there would not create more physical RAM, and has not been validated as a fix. -4. A passing process-leak audit cannot detect retained files or predict shared-memory allocation spikes. Browser retries start fresh processes but inherit the same worker's persisted files and limited capacity. - -## Other intermittent tests - -The reported demo-mode case was on **shard 36**, not shard 8. `summary.json` incorrectly points its log to shard 8; the native shard plan and report are the stronger evidence. Its first attempt timed out in `afterEach`, during cleanup of its server/site; retry passed in 1.394 s. This is **not yet attributable** to Chromium resource exhaustion. - -Both `browser-demo-mode.test.ts` and `channel-switcher-context-menu.test.ts` use the unbundled development application; demo-mode also reloads it. They share exposure to the capacity problem, but no trace or kernel evidence was supplied for the channel-switcher drag failure, and this investigation has not established that either intermittent failure shares the cause. Do not relabel all browser flakes as resource failures. - -## Recommended fix and scope - -1. **Fix the test-runtime file leak and reclaim existing accumulation on idle leased workers.** Give the source-CLI runtime a worker lifetime, clean it after its product servers stop, and cover Playwright and other callers of the helper. Preserve once-per-worker build reuse; deleting it after every test would add unnecessary builds. Reclaim demonstrably orphaned generated runtime directories, rather than arbitrary `/tmp` files or processes selected by name. This is test-harness/provider work, with no production behavior change. -2. **Use the built application for product acceptances that do not require development-module behavior.** The PR #9 author's theme-reset change removes the triggering fan-out. Review other `ProductServer.appURL()` consumers for the same distinction; component fixtures that intentionally exercise source modules may need different handling. This is test setup work, not a production bundling change. -3. **Make failures diagnosable.** Retain small per-attempt memory/free-space measurements and OOM-counter changes, plus browser crashes/request errors, when acceptance fails. Process-leak checks alone cannot identify this failure class. Bound artifact retention on persistent workers. - -Increasing retries, globally serializing the suite, or raising descriptor limits is not the primary fix: the failures already occur in one test with one worker and low descriptor counts. More RAM or fresh workers provide capacity, but do not cure runtime-directory retention. Routine forced GC is not the proposed fix. - -There is also a separate reproducibility issue: the coordinator aliases Chromium build 1223 into the build-1208 path expected by Playwright 1.58.2. Its manifest expects Chromium `145.0.7632.6`, while tests actually run `148.0.7778.96`. Provision the matching browser version instead of this alias. Whether version alignment changes buffer retention remains untested; it is not established as the root cause. - -## What remains unknown - -- The exact failed allocation syscall/errno for the original 16 SVG errors. The response-pipe path is inferred from source, accepted HTTP responses, capacity measurements, and the GC experiment. -- Whether Chromium 145 retains the same buffers, and whether Chromium 148's retention is intended GC timing or a regression. -- Whether cleanup alone leaves sufficient headroom for every source-based acceptance on an 8 GB worker. No cleanup-only A/B or post-fix full verify was performed. -- Whether demo-mode teardown or channel-switcher drag flakes share any part of this cause. -- The original worker's exact capacity at the historical failure instant; later inspection cannot recover it. - -## Reproduction and retained evidence - -Experiment revisions remain in branch history. The final report revision restores test/harness files to `68b8ab11`; instrumentation and forced GC are not proposed changes for merge. - -Targeted runs used: - -```sh -npm test -- blaxel --file packages/web/test/e2e/theme-product-acceptance.test.ts --grep 'executable theme reset reloads connected Chromium documents under confirmed state' --ignore-uncommitted --run-dir-output /tmp/rose-chromium-runN -``` - -Experiment 2 omitted `--grep`. Experiment 3 used: - -```sh -npm test -- blaxel --shard-indices 8 --force --ignore-uncommitted --run-dir-output /tmp/rose-chromium-run3 -``` - -`--ignore-uncommitted` acknowledged only a local `node_modules` symlink to already-installed coordinator dependencies. Actual test code was committed and pushed. The first attempt to dispatch experiment 1 stopped at remote preflight because this symlink appeared untracked; it ran no tests. - -[measurements.json](chromium-insufficient-resources-evidence/measurements.json) retains compact measurements, commit/sandbox/run-directory identities, and GC snapshots. Detailed [GC renderer descriptors](chromium-insufficient-resources-evidence/gc-renderers.json) and [original-worker filesystem inspection](chromium-insufficient-resources-evidence/filesystem-inspection.txt) are also retained. Kernel evidence: [shard replay](chromium-insufficient-resources-evidence/run3-kernel-oom.txt), [isolated case](chromium-insufficient-resources-evidence/run4-kernel-oom.txt). Full normalized reports remain under this worktree's `.test-runs/`, with exact paths in the JSON. Downloaded NetLogs remain at `/tmp/rose-run1.netlog.json` and `/tmp/rose-run3-retry2.netlog.json` on the development host. - -Primary Chromium 148 source references: - -- [Response buffer size: 2 MiB](https://github.com/chromium/chromium/blob/148.0.7778.96/services/network/public/cpp/loading_params.cc#L15-L45). -- [Response-pipe allocation failure becomes ERR_INSUFFICIENT_RESOURCES](https://github.com/chromium/chromium/blob/148.0.7778.96/services/network/url_loader.cc#L1216-L1238). -- [DataPipe allocates shared memory](https://github.com/chromium/chromium/blob/148.0.7778.96/mojo/core/ipcz_driver/data_pipe.cc#L145-L170). -- [Shared-memory files and AllocateFileRegion](https://github.com/chromium/chromium/blob/148.0.7778.96/base/memory/platform_shared_memory_region_posix.cc#L189-L243). -- [Outstanding-loader limit: 2,700](https://github.com/chromium/chromium/blob/148.0.7778.96/services/network/network_context.h#L968-L972) and [admission check](https://github.com/chromium/chromium/blob/148.0.7778.96/services/network/url_loader_factory.cc#L238-L283). -- [POSIX descriptor errors also map to this code](https://github.com/chromium/chromium/blob/148.0.7778.96/net/base/net_errors_posix.cc#L84-L113), explaining why the error string alone cannot identify the resource. diff --git a/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03-sizes.json b/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03-sizes.json deleted file mode 100644 index 704f61d3..00000000 --- a/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03-sizes.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "sandbox": "tv-testshard-x64-poc-03", - "capturedAt": "2026-10-01T22:46:47.023Z", - "command": "du -B1 --max-depth=1 /tmp", - "sizes": { - "/tmp/node-compile-cache": 8892416, - "/tmp/television-source-cli-e2e-*": 55930880, - "/tmp/television-web-bundle-*": 22622208, - "/tmp/tv-attest-cli-*": 8613888, - "/tmp/tv-attest-prune-*": 8859648, - "/tmp/tv-attest-rundir-*": 24576, - "/tmp/tv-attest-writer-*": 3661824, - "/tmp/tv-attest-clone-*": 393216, - "/tmp/tv-toolchain-npm-*": 8192, - "/tmp/tv-toolchain-policy-mismatch-*": 8192, - "/tmp/tv-toolchain-policy-*": 8192, - "/tmp/tv-test-registry-*": 24576, - "/tmp/tv-provider-reports-*": 12288, - "/tmp/tv-provider-corrupt-*": 4096, - "/tmp/tv-provider-combine-*": 0, - "/tmp/tv-provider-stale-owner-*": 0, - "/tmp/tv-provider-leak-*": 4096, - "/tmp/tv-provider-remote-native-*": 4096, - "/tmp/tv-provider-blaxel-wrapper-*": 4096, - "/tmp/tv-provider-timing-shards-*": 4096, - "/tmp/tv-provider-normalize-*": 8192, - "/tmp/tv-posthog-preflight-missing-*": 0, - "/tmp/tv-posthog-preflight-*": 4096, - "/tmp/.X11-unix": 0, - "/tmp/playwright-transform-cache-0": 19845120, - "/tmp/multipart-uploads": 0, - "/tmp": 156442624 - }, - "exitCode": 0 -} diff --git a/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03.json b/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03.json deleted file mode 100644 index 9f3a22f5..00000000 --- a/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03.json +++ /dev/null @@ -1,420 +0,0 @@ -{ - "capturedAt": "2026-10-01T22:44:20.524241+00:00", - "tmp": { - "availableBytes": 3700719616, - "totalBytes": 6676103168, - "categories": [ - { - "prefix": ".X*", - "count": 1, - "examples": [ - { - "name": ".X11-unix", - "kind": "dir" - } - ] - }, - { - "prefix": ".org.chromium.Chromium.*", - "count": 9, - "examples": [ - { - "name": ".org.chromium.Chromium.BOuDBO", - "kind": "file" - }, - { - "name": ".org.chromium.Chromium.W2lXu2", - "kind": "file" - } - ] - }, - { - "prefix": "multipart-uploads", - "count": 1, - "examples": [ - { - "name": "multipart-uploads", - "kind": "dir" - } - ] - }, - { - "prefix": "node-compile-cache*", - "count": 1, - "examples": [ - { - "name": "node-compile-cache", - "kind": "dir" - } - ] - }, - { - "prefix": "playwright-*", - "count": 1, - "examples": [ - { - "name": "playwright-transform-cache-0", - "kind": "dir" - } - ] - }, - { - "prefix": "rose-*", - "count": 2, - "examples": [ - { - "name": "rose-chromium-11632.netlog.json", - "kind": "file" - }, - { - "name": "rose-chromium-11632.log", - "kind": "file" - } - ] - }, - { - "prefix": "television-source-cli-e2e-*", - "count": 17, - "examples": [ - { - "name": "television-source-cli-e2e-6bIaZE", - "kind": "dir" - }, - { - "name": "television-source-cli-e2e-P8NxgI", - "kind": "dir" - } - ] - }, - { - "prefix": "television-web-bundle-*", - "count": 27, - "examples": [ - { - "name": "television-web-bundle-0TE0fq", - "kind": "dir" - }, - { - "name": "television-web-bundle-1xECmr", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-attest-cli-*", - "count": 39, - "examples": [ - { - "name": "tv-attest-cli-38qL1c", - "kind": "dir" - }, - { - "name": "tv-attest-cli-6NzZvS", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-attest-clone-*", - "count": 3, - "examples": [ - { - "name": "tv-attest-clone-73s4Yq", - "kind": "dir" - }, - { - "name": "tv-attest-clone-g2egxA", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-attest-prune-*", - "count": 33, - "examples": [ - { - "name": "tv-attest-prune-1Kbfdh", - "kind": "dir" - }, - { - "name": "tv-attest-prune-36WSdp", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-attest-rundir-*", - "count": 3, - "examples": [ - { - "name": "tv-attest-rundir-IudtUg", - "kind": "dir" - }, - { - "name": "tv-attest-rundir-Su49lM", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-attest-writer-*", - "count": 18, - "examples": [ - { - "name": "tv-attest-writer-51Dp9C", - "kind": "dir" - }, - { - "name": "tv-attest-writer-7XgAsh", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-git-askpass.sh", - "count": 1, - "examples": [ - { - "name": "tv-git-askpass.sh", - "kind": "file" - } - ] - }, - { - "prefix": "tv-owner-cleanup-*", - "count": 1, - "examples": [ - { - "name": "tv-owner-cleanup-2026-09-29T18-41-53-559Z-1385612-shard-3.json", - "kind": "file" - } - ] - }, - { - "prefix": "tv-posthog-preflight-*", - "count": 1, - "examples": [ - { - "name": "tv-posthog-preflight-6zpiaH", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-posthog-preflight-missing-*", - "count": 1, - "examples": [ - { - "name": "tv-posthog-preflight-missing-4tzN0y", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-blaxel-wrapper-*", - "count": 1, - "examples": [ - { - "name": "tv-provider-blaxel-wrapper-soo5KV", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-combine-*", - "count": 1, - "examples": [ - { - "name": "tv-provider-combine-XnpJi1", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-corrupt-*", - "count": 1, - "examples": [ - { - "name": "tv-provider-corrupt-3nq4hS", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-leak-*", - "count": 1, - "examples": [ - { - "name": "tv-provider-leak-VXhPIP", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-normalize-*", - "count": 1, - "examples": [ - { - "name": "tv-provider-normalize-Ky71z0", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-remote-native-*", - "count": 1, - "examples": [ - { - "name": "tv-provider-remote-native-H1RqSp", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-reports-*", - "count": 1, - "examples": [ - { - "name": "tv-provider-reports-Y1Nd1C", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-stale-owner-*", - "count": 1, - "examples": [ - { - "name": "tv-provider-stale-owner-9WjHLo", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-timing-shards-*", - "count": 1, - "examples": [ - { - "name": "tv-provider-timing-shards-Zv3xyW", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-test-registry-*", - "count": 3, - "examples": [ - { - "name": "tv-test-registry-ESqI4b", - "kind": "dir" - }, - { - "name": "tv-test-registry-la3yV1", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-testshard-*", - "count": 192, - "examples": [ - { - "name": "tv-testshard-2026-10-01T22-21-05-004Z-2883021-s1.sh", - "kind": "file" - }, - { - "name": "tv-testshard-2026-10-01T22-04-01-804Z-2873890-s3.sh", - "kind": "file" - } - ] - }, - { - "prefix": "tv-toolchain-npm-*", - "count": 2, - "examples": [ - { - "name": "tv-toolchain-npm-5sTMQk", - "kind": "dir" - }, - { - "name": "tv-toolchain-npm-Fmnb69", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-toolchain-policy-*", - "count": 1, - "examples": [ - { - "name": "tv-toolchain-policy-egmSc5", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-toolchain-policy-mismatch-*", - "count": 1, - "examples": [ - { - "name": "tv-toolchain-policy-mismatch-T9vBXh", - "kind": "dir" - } - ] - } - ] - }, - "meminfo": { - "MemTotal": "8149540 kB", - "MemAvailable": "4670476 kB", - "SwapTotal": "0 kB", - "SwapFree": "0 kB", - "Shmem": "2906032 kB" - }, - "lock": null, - "processes": [ - { - "pid": 1, - "command": "init", - "ownerTokens": [] - }, - { - "pid": 424, - "command": "ukp-fs", - "ownerTokens": [] - }, - { - "pid": 427, - "command": "entrypoint.sh", - "ownerTokens": [] - }, - { - "pid": 428, - "command": "sandbox-api", - "ownerTokens": [] - }, - { - "pid": 429, - "command": "MainThread", - "ownerTokens": [] - }, - { - "pid": 11818, - "command": "sh", - "ownerTokens": [] - }, - { - "pid": 11819, - "command": "python3", - "ownerTokens": [] - }, - { - "pid": 11820, - "command": "sleep", - "ownerTokens": [] - } - ], - "tmpReferences": [], - "sandbox": "tv-testshard-x64-poc-03", - "method": "Read-only os.scandir(/tmp), statvfs, selected /proc/meminfo values, process comm and TV_TEST_SURFACE_OWNER only, cwd/fd paths under /tmp, and the shard owner.json if present. Random six-character directory suffixes grouped for readability. No file contents or environment secrets collected." -} diff --git a/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131-sizes.json b/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131-sizes.json deleted file mode 100644 index dff646f1..00000000 --- a/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131-sizes.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "sandbox": "tv-testshard-x64-poc-131", - "capturedAt": "2026-10-01T22:46:46.362Z", - "command": "du -B1 --max-depth=1 /tmp", - "sizes": { - "/tmp/node-compile-cache": 12963840, - "/tmp/playwright-artifacts-*": 0, - "/tmp/television-source-cli-e2e-*": 1059090432, - "/tmp/television-web-bundle-*": 104255488, - "/tmp/xvfb-run.9dfpKV": 4096, - "/tmp/tv-electron-env-*": 565248, - "/tmp/tv-electron-missing-*": 12288, - "/tmp/tv-dynamic-services-*": 32768, - "/tmp/tv-blaxel-token-*": 12288, - "/tmp/.X11-unix": 0, - "/tmp/tv-provider-reports-*": 49152, - "/tmp/tv-provider-contradiction-*": 12288, - "/tmp/tv-provider-corrupt-*": 12288, - "/tmp/tv-provider-combine-*": 0, - "/tmp/tv-provider-stale-owner-*": 0, - "/tmp/tv-provider-leak-*": 12288, - "/tmp/tv-provider-remote-native-*": 12288, - "/tmp/tv-provider-blaxel-wrapper-*": 12288, - "/tmp/tv-provider-task-infra-*": 12288, - "/tmp/tv-provider-distinct-infra-*": 0, - "/tmp/tv-provider-timing-shards-*": 12288, - "/tmp/tv-provider-normalize-*": 24576, - "/tmp/tv-provider-empty-*": 0, - "/tmp/tv-partial-leases-*": 24576, - "/tmp/tv-interrupted-attempt-*": 86016, - "/tmp/playwright-transform-cache-0": 17170432, - "/tmp/tsx-0": 589824, - "/tmp/tv-posthog-preflight-missing-*": 0, - "/tmp/tv-posthog-preflight-*": 4096, - "/tmp/multipart-uploads": 0, - "/tmp": 1197846528 - }, - "exitCode": 0 -} diff --git a/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131.json b/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131.json deleted file mode 100644 index 9f43a4bc..00000000 --- a/docs/working/test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131.json +++ /dev/null @@ -1,502 +0,0 @@ -{ - "capturedAt": "2026-10-01T22:44:17.626697+00:00", - "tmp": { - "availableBytes": 2137575424, - "totalBytes": 6676103168, - "categories": [ - { - "prefix": ".X*", - "count": 1, - "examples": [ - { - "name": ".X11-unix", - "kind": "dir" - } - ] - }, - { - "prefix": ".org.chromium.Chromium.*", - "count": 3, - "examples": [ - { - "name": ".org.chromium.Chromium.p98fYu", - "kind": "file" - }, - { - "name": ".org.chromium.Chromium.SPumEs", - "kind": "file" - } - ] - }, - { - "prefix": "MozillaUpdateLock-847956029D21FDC7", - "count": 1, - "examples": [ - { - "name": "MozillaUpdateLock-847956029D21FDC7", - "kind": "file" - } - ] - }, - { - "prefix": "multipart-uploads", - "count": 1, - "examples": [ - { - "name": "multipart-uploads", - "kind": "dir" - } - ] - }, - { - "prefix": "node-compile-cache*", - "count": 1, - "examples": [ - { - "name": "node-compile-cache", - "kind": "dir" - } - ] - }, - { - "prefix": "playwright-*", - "count": 2, - "examples": [ - { - "name": "playwright-artifacts-5Qz2J3", - "kind": "dir" - }, - { - "name": "playwright-transform-cache-0", - "kind": "dir" - } - ] - }, - { - "prefix": "television-source-cli-e2e-*", - "count": 267, - "examples": [ - { - "name": "television-source-cli-e2e-0UvOF0", - "kind": "dir" - }, - { - "name": "television-source-cli-e2e-xz6UWF", - "kind": "dir" - } - ] - }, - { - "prefix": "television-web-bundle-*", - "count": 146, - "examples": [ - { - "name": "television-web-bundle-0U3Bq1", - "kind": "dir" - }, - { - "name": "television-web-bundle-0zVhkg", - "kind": "dir" - } - ] - }, - { - "prefix": "tsx-*", - "count": 1, - "examples": [ - { - "name": "tsx-0", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-blaxel-token-*", - "count": 7, - "examples": [ - { - "name": "tv-blaxel-token-7V3eTY", - "kind": "dir" - }, - { - "name": "tv-blaxel-token-FFrYeT", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-dynamic-services-*", - "count": 8, - "examples": [ - { - "name": "tv-dynamic-services-Cvuced", - "kind": "dir" - }, - { - "name": "tv-dynamic-services-L4ir31", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-electron-env-*", - "count": 33, - "examples": [ - { - "name": "tv-electron-env-0RIXYm", - "kind": "dir" - }, - { - "name": "tv-electron-env-0tSmO7", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-electron-missing-*", - "count": 3, - "examples": [ - { - "name": "tv-electron-missing-OoUqnq", - "kind": "dir" - }, - { - "name": "tv-electron-missing-Y711Eu", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-git-askpass.sh", - "count": 1, - "examples": [ - { - "name": "tv-git-askpass.sh", - "kind": "file" - } - ] - }, - { - "prefix": "tv-interrupted-attempt-*", - "count": 3, - "examples": [ - { - "name": "tv-interrupted-attempt-M63Tzl", - "kind": "dir" - }, - { - "name": "tv-interrupted-attempt-NqzTGZ", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-owner-cleanup-*", - "count": 2, - "examples": [ - { - "name": "tv-owner-cleanup-2026-09-29T18-41-53-559Z-1385612-shard-8.json", - "kind": "file" - }, - { - "name": "tv-owner-cleanup-2026-09-11T19-44-47-497Z-2030667-shard-8.json", - "kind": "file" - } - ] - }, - { - "prefix": "tv-partial-leases-*", - "count": 3, - "examples": [ - { - "name": "tv-partial-leases-5Gl2yt", - "kind": "dir" - }, - { - "name": "tv-partial-leases-TZE8QZ", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-posthog-preflight-*", - "count": 1, - "examples": [ - { - "name": "tv-posthog-preflight-N3e9KL", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-posthog-preflight-missing-*", - "count": 1, - "examples": [ - { - "name": "tv-posthog-preflight-missing-utVaiY", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-blaxel-wrapper-*", - "count": 3, - "examples": [ - { - "name": "tv-provider-blaxel-wrapper-YbDL1t", - "kind": "dir" - }, - { - "name": "tv-provider-blaxel-wrapper-s1eNDm", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-combine-*", - "count": 3, - "examples": [ - { - "name": "tv-provider-combine-KWaBaw", - "kind": "dir" - }, - { - "name": "tv-provider-combine-NMu42s", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-contradiction-*", - "count": 3, - "examples": [ - { - "name": "tv-provider-contradiction-B59JgY", - "kind": "dir" - }, - { - "name": "tv-provider-contradiction-JwJxXQ", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-corrupt-*", - "count": 3, - "examples": [ - { - "name": "tv-provider-corrupt-8J3HdF", - "kind": "dir" - }, - { - "name": "tv-provider-corrupt-FPlCy3", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-distinct-infra-*", - "count": 3, - "examples": [ - { - "name": "tv-provider-distinct-infra-TNDzyX", - "kind": "dir" - }, - { - "name": "tv-provider-distinct-infra-r9YzK1", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-empty-*", - "count": 3, - "examples": [ - { - "name": "tv-provider-empty-SfDSYb", - "kind": "dir" - }, - { - "name": "tv-provider-empty-ar7rau", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-leak-*", - "count": 3, - "examples": [ - { - "name": "tv-provider-leak-0FzFHu", - "kind": "dir" - }, - { - "name": "tv-provider-leak-UF5ev3", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-normalize-*", - "count": 3, - "examples": [ - { - "name": "tv-provider-normalize-KGv4so", - "kind": "dir" - }, - { - "name": "tv-provider-normalize-Odm1iY", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-remote-native-*", - "count": 3, - "examples": [ - { - "name": "tv-provider-remote-native-LhC04g", - "kind": "dir" - }, - { - "name": "tv-provider-remote-native-akc5RU", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-reports-*", - "count": 3, - "examples": [ - { - "name": "tv-provider-reports-0Jfsvl", - "kind": "dir" - }, - { - "name": "tv-provider-reports-12iCrO", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-stale-owner-*", - "count": 3, - "examples": [ - { - "name": "tv-provider-stale-owner-BaHmEj", - "kind": "dir" - }, - { - "name": "tv-provider-stale-owner-OX1kvE", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-task-infra-*", - "count": 3, - "examples": [ - { - "name": "tv-provider-task-infra-4VzJKa", - "kind": "dir" - }, - { - "name": "tv-provider-task-infra-QOrMRp", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-provider-timing-shards-*", - "count": 3, - "examples": [ - { - "name": "tv-provider-timing-shards-GD7UrH", - "kind": "dir" - }, - { - "name": "tv-provider-timing-shards-P7dmtU", - "kind": "dir" - } - ] - }, - { - "prefix": "tv-testshard-*", - "count": 222, - "examples": [ - { - "name": "tv-testshard-2026-10-01T22-04-01-804Z-2873890-s8.sh", - "kind": "file" - }, - { - "name": "tv-testshard-2026-10-01T21-41-24-129Z-2861191-s8.sh", - "kind": "file" - } - ] - }, - { - "prefix": "xvfb-run.*", - "count": 1, - "examples": [ - { - "name": "xvfb-run.9dfpKV", - "kind": "dir" - } - ] - } - ] - }, - "meminfo": { - "MemTotal": "8149540 kB", - "MemAvailable": "3077332 kB", - "SwapTotal": "0 kB", - "SwapFree": "0 kB", - "Shmem": "4433052 kB" - }, - "lock": null, - "processes": [ - { - "pid": 1, - "command": "init", - "ownerTokens": [] - }, - { - "pid": 423, - "command": "ukp-fs", - "ownerTokens": [] - }, - { - "pid": 426, - "command": "entrypoint.sh", - "ownerTokens": [] - }, - { - "pid": 427, - "command": "sandbox-api", - "ownerTokens": [] - }, - { - "pid": 428, - "command": "MainThread", - "ownerTokens": [] - }, - { - "pid": 16872, - "command": "sh", - "ownerTokens": [] - }, - { - "pid": 16873, - "command": "python3", - "ownerTokens": [] - }, - { - "pid": 16874, - "command": "sleep", - "ownerTokens": [] - } - ], - "tmpReferences": [], - "sandbox": "tv-testshard-x64-poc-131", - "method": "Read-only os.scandir(/tmp), statvfs, selected /proc/meminfo values, process comm and TV_TEST_SURFACE_OWNER only, cwd/fd paths under /tmp, and the shard owner.json if present. Random six-character directory suffixes grouped for readability. No file contents or environment secrets collected." -} diff --git a/docs/working/test-harness-resource-cleanup-integrated.md b/docs/working/test-harness-resource-cleanup-integrated.md deleted file mode 100644 index c39e0caa..00000000 --- a/docs/working/test-harness-resource-cleanup-integrated.md +++ /dev/null @@ -1,169 +0,0 @@ -# Test-harness resource cleanup: integrated validation - -Status: independent integrated review passed at `30774259`; the contribution has converged. Pre-merge docs preparation and ready-PR CI follow. -Owner: rose. Draft PR #12. Branch: `thopter/test-harness-resource-cleanup`. -No test, build, lint, type check, browser or server runs on the shared development -host. The dedicated Blaxel coordinator performs all execution below. - -## Target reconciliation - -On 2026-10-27, fetched `origin` and read the complete branch histories on both sides. -`origin/main` is `a8e73fbf5eecfb27172ff5928ffb71f0c6844368`, already an ancestor -of the reviewed branch head `dc1d9f2e`. The merge base is that same main commit; -there are no incoming commits to merge. The earlier real merge is `74d22941`. -The logical-merging skill was read in full. Its policy-level reconciliation still -applies; an artificial merge adds no integration evidence. - -Audited the complete changed-path inventory against main and the governing specs, -proofs and converged slices. There are no production application changes. Checked -the interaction paths beyond changed files: - -- Worker build ownership spans shared Playwright/Vitest fixtures and direct - product-server/versioned-bundle callers. Failed preparation is removed; pending - preparation and servers settle before reusable worker outputs are disposed. - Version-keyed reuse and serialized web builds remain intact. -- The five built-application conversions preserve same-origin authentication and - stable-front restart continuity. PR #9's built reset and the separate Settings - development walk remain. Main's demo fixture connection teardown remains. -- The Vite service still strips conditional validators before its middleware, - preserving main's full-response correction. This contribution claims no measured - incremental memory saving after that correction. -- Parent diagnostics and bounded retained links cross planned and targeted paths, - provider normalization, retry aggregation and final reporting. Owner attribution - and `TMPDIR` survive restricted child environments. -- Reservation, dispatch registration, teardown, retrieval, recovery and pool - publication/deletion retain the converged exclusion rules. The default recovery - interval is 1200 seconds for every profile. Existing execution deadlines and - native/infrastructure retry budgets remain. Main's origin-repository checkout - selection is unchanged; result-reset preservation remains covered by slice 3. -- The two harness experiment surfaces are excluded from ordinary `all` and CI. - The committed private driver and every new live proof obligation have evidence; - no new pending proof citation remains. Intentional fixture failures are retained - as failures, separate from the passing outer qualification. - -No textual conflict, policy conflict or additional behavior edit was found. The -only code-file change after slice convergence is the review-requested dry-run -comment. The full gate validates the composition at the pushed checkpoint. - -## Slice 3 review observations - -[Round 2](test-harness-resource-cleanup-slice-3-review-r2.md) passes the slice. A -comment now explains that clearing the inherited shard token is limited to the -two immediate dry-run CLI placement cases; ordinary surface ownership remains. -Record both the number of shard attempts and distinct shard indices reporting -`Lease renewal lost` in the full-gate result. Fail-on-first-renewal-error behavior -is unchanged; do not infer transport tolerance from the private qualification. - -## Private cleanup and full gate - -The passing live driver deleted `tv-testshard-x64-rose-cleanup-01` and `-02` through -the canonical pool manager. All its evidence is retained locally. Read-only API -inventory at 2026-10-27 21:26:06 UTC finds only `tv-cleanup-coordinator-rose` among -those three private resources. It reports 144 deployed sandboxes overall; this is -an inventory, not a quota or active-concurrency measurement. - -At pushed checkpoint `7c66d160c2494bb673e16010563f0c8b5887f36e`, the canonical -private-pool delete and list both exited zero with no workers listed. The dedicated -coordinator remained outside the worker-pool labels and then executed: - -```sh -npm test -- pool blaxel delete --pool rose-cleanup -npm test -- pool blaxel list --pool rose-cleanup -npm run verify -- blaxel --pool poc -``` - -The gate used all standard 36 shards and the default native/infrastructure retry -budgets of two. No fault experiment, provisioning or deletion targeted shared -workers. Normal lease acquisition and cleanup ran under the converged rules. - -## Full-gate result - -`npm run verify -- blaxel --pool poc` exited **0** at `7c66d160`: - -- All 36 shard indices passed on infrastructure attempt 1. All 22 selected - surfaces ran and passed, covering 370 assigned files. Aggregated native counts: - **3,705 passed, 8 skipped, 0 failed, 0 recovered retries** (3,713 total). -- **0 process leaks and 0 infrastructure failures.** `Lease renewal lost`: - **0 shard attempts, 0 distinct shard indices**, checked against the complete - provider report, attempt records and logs. This one run does not establish a - transport-error tolerance guarantee or a long-term renewal failure rate. -- Lint, type checking, package manifests and the final vibe-mode check passed. - `verify:total` was 141,073 ms; the test run was 119,945 ms. Test execution ran - from 21:30:30.831 to 21:32:30.776 UTC on 2026-10-27. -- Retained 231 resource sidecars, all linked from the final surface results and - present on disk, each with observed start/end samples. Every OOM-kill delta was - zero. There were no native failed attempts, so kernel failure excerpts were - correctly not requested. Largest sidecar: 1,741 bytes; largest combined resource - sidecars for one shard: 13,727 bytes. These are interval observations, not peaks. -- The 36 pre-run cleanup reports show 1,013 source-CLI directories and 497 web - bundle directories removed, with no cleanup skips/errors. Their recorded - allocated bytes sum to **5,003,157,504 bytes** (about 5.0 GB / 4.66 GiB), equal - to the summed before/after available-space delta. This is actual legacy residue - reclaimed during ordinary acquisition, not a seeded shared-worker experiment. - -Four acquisition candidates were skipped because -`/home/playwright/.nvm/nvm.sh` was absent: `tv-testshard-x64-poc-101`, `-08`, `-109` -and `-12`. The runner acquired replacements and every planned shard completed. -Those skipped workers were not repaired or reconfigured. Their missing runtime -cause remains unknown and is independent of the passing selected-shard results. - -Automatic attestation publication was attempted after verification and failed -non-fatally: GitHub returned HTTP 403, permission denied to `jwhiting`, during the -ref update/fallback fetch. **The gate passed; no attestation publication is -claimed.** Ready-PR CI must therefore run its heavy jobs in full rather than reuse this Blaxel result through a `refs/testpass` attestation. No credential changes, attestation retry or bypass was attempted. - -A fresh `git fetch origin` after the gate still resolves main to `a8e73fbf`. -The tested tree contains that full target. Subsequent record edits change only -working documents; `7c66d160` is the exact tested revision, not an assertion that -any later documentation commit ran another gate. - -## Retained evidence - -The complete run is -[`.test-runs/2026-10-27T21-30-30-229Z-p40561-rde511175732d79a8/summary.json`](../../.test-runs/2026-10-27T21-30-30-229Z-p40561-rde511175732d79a8/summary.json). -Its `results.json`, `provider/blaxel/report.json`, provider summary, all 36 shard -logs/reports, native output and cleanup/resource sidecars are retained beneath -that directory. SDK binary downloads copied **1,100 files / 23,423,163 bytes**; -each downloaded length matched the coordinator filesystem inventory. - -Additional local evidence is under `.test-runs/integrated-validation/`: -`rose-integrated-verify.log` (including private-pool absence and the attestation -error), `download-inventory.json`, `metrics.json`, and `private-cleanup.json`. -No test or static check executed on the shared development host. - -After retaining evidence, the coordinator's API listed every dispatched process -as completed or failed, with the full gate completed/exit 0. Requested deletion -of the dedicated coordinator only. Read-only API inventory at -**2026-10-27 21:36:31 UTC** confirms all three private resource names absent -(143 remaining sandboxes, all deployed). The two private workers had already -been deleted. No shared worker was deleted. - -## Remaining work and limits - -Independent integrated review is next. Docs preparation, PR-ready status, Josh's -spec review and required GitHub checks remain later obligations. The attestation -write did not succeed. The four skipped workers' runtime problem is unresolved. -Incremental memory savings after the Vite correction and long-term renewal-loss -frequency remain outside this evidence. - -No new lasting decision was made in this integration step. Primitive's commit -hook listed eight existing decisions, including superseded conversation wording, -and reported a truncated scan. This is not a complete governing check; the user -corrections and converged specs remain authoritative. The earlier unread warning -remains in the slice record; its CLI received no further debugging. - - -## Integrated review refinements - -The independent integrated review passed at `30774259`. Removed the two unused -lease-module constants and their declarations, corrected the stale pending-proof -sentence, and commented the release helper's interruption flag without changing -its behavior. The attestation paragraph now states the consequence for ready-PR -CI. No additional live qualification is needed for these bounded corrections. - -The stale pool-size example and four workers with missing nvm remain visible -observations for human spec review and operations; this contribution does not -repair or resize the shared pool. The zero-renewal-loss gate remains one observation, -not a long-term reliability rate. Pre-merge docs prep will retain the proposal and -plan as coherent design records, delete completed checkpoint/review records, and -carry essential validation facts and limitations into the PR description. diff --git a/docs/working/test-harness-resource-cleanup-plan-review-r1.md b/docs/working/test-harness-resource-cleanup-plan-review-r1.md deleted file mode 100644 index fa346984..00000000 --- a/docs/working/test-harness-resource-cleanup-plan-review-r1.md +++ /dev/null @@ -1,61 +0,0 @@ -# Plan review, round 1: test-harness resource cleanup (PR #12, 70f27ccb) - -**Verdict: FAIL, on one item.** The three slices are coherent, every one of the 31 pending proof obligations has a slice, the red/green criteria and intermediate baselines are honest, the scope stays inside the converged specs, and nothing in the plan runs on the shared development host. The blocker is the final gate. The plan moves full verification from the shared `poc` pool, where the converged proposal put it, to the two private workers with `--shards 2`. The repository's own timing baseline says each of those two shards would carry about 441 seconds of measured test compute before any setup, against the 480-second worker timeout the canonical runner cannot raise, on a tier that cannot hold a larger private pool. The gate as planned will not complete, and the plan's contingency is to stop and report at the end of the work rather than decide now. Returning the gate to `poc` at the standard shard count, run from the dedicated coordinator sandbox, clears it in a sentence. Four refinements follow, none blocking. - -- Reviewed: `docs/working/test-harness-resource-cleanup-plan.md` at `70f27ccb` against the workflow's Planning and slices section, the testing policy's iteration and verification rules, the runner spec's publication-qualification and verify rules, the reviewer checklist, the proof obligations at this revision, the runner's option forwarding, and the committed timing baseline. Also the bounded proof binding and two spec clauses since `3fa15d47`. -- Branch state: contains `origin/main` (`a8e73fbf`); the copied round 2 proof review is identical to mine. -- I ran nothing. I read `test/timing-baseline.json` with a small script that sums its recorded medians; nothing was built, tested or changed. - -## The binding since round 2 stayed in scope - -The shared live declaration now names both surfaces, the coordinator host rule and both selection commands, the coverage model names the placement, and the registry proof carries one source-level selection contract (`^registry-t-blaxel-harness-selection`). The runner spec gained the `Selectors` clause and the guardrail cross-reference I asked for. Nothing else moved. The one unrelated pending anchor in the tree (`proofs/arch/cli/startup-bind-failure.md`) predates this branch. - -## Slices, obligations, baselines - -| Slice | Obligations placed | Baseline stated | -| --- | --- | --- | -| 1: worker-owned builds and the five built-application walks | `^testing-t-build-lifetime`, `^testing-t-build-worker-seam`; the eleven theme, delivery, artifact and bridge citations graded "built-application boundary: test to be written" | Green for its obligations; killed-worker backstop assigned to slice 3 (`plan.md:37`). | -| 2: bounded diagnostics and report transport | `^reporting-t-resource-bounds`, `-parent`, `-links` | Explicit: `^reporting-t-resource-live` pending, lease/download race still present, checkpoint not safe for concurrent reuse (`:51`). | -| 3: exclusive lifecycle, scratch, pool mutation, live qualification | `^blaxel-t-lease-decisions`, `-lease-order`, `-worker-guard`, `-scratch-cleanup`, `-scratch-errors`, `-pool-order`, the `^blaxel-pool-pagination` extension, `^registry-t-blaxel-harness-selection`, and the five live assertions including `^reporting-t-resource-live` | Resolves every earlier baseline (`:74`). | - -The slice judgment is sound: helper ownership and shipped-asset loading have no dependence on the coordinator; diagnostics have their own native-runner boundary; and the lease, dispatch, recovery, scratch and pool rules are one safety argument that should not be reviewed in halves. Three slices is the right count. The order lets the live runs prove the composition once. Each slice's red/green discipline is the policy's (narrow red first, widen after green, a failed import is not red), the existing TTL assertions are replaced rather than left endorsing superseded behavior, and the plan keeps the retry defaults, shard default and browser version out of scope. - -Execution placement is right throughout: edit, commit and push on the shared host; every run on Blaxel through the canonical runner from `tv-cleanup-coordinator-rose`; the driver as `npm test -- local --surface experiment:blaxel-harness` on that sandbox; private pool `rose-cleanup` for everything that touches workers; the pool deleted and re-provisioned around the fresh-provisioning acceptance because `ensure` is fresh-only. - -## Blocker: the final gate cannot complete as planned - -`plan.md:80-88` runs `npm run verify -- blaxel --pool rose-cleanup --shards 2 --allow-shard-count-override 1` on the two private workers and says this "replaces the proposal's earlier shared-`poc` final-run plan", with no reason given. It qualifies under the runner spec (`--shards` repartitions without narrowing, `test-runner.md:76`) and `verify` forwards `--pool`, `--shards` and the override (`scripts/test/cli.mjs:284`). It is still the wrong gate: - -- **Compute.** `test/timing-baseline.json` records Blaxel medians for 311 files totalling about 883 seconds, which is the 25-second mean per shard at 36 shards the Blaxel spec quotes. Two shards carry about 441 seconds each before checkout, dependency, pre-command builds, readiness and any e2e retry. `e2e:browser-app` alone is 376 seconds. -- **Ceiling.** The coordinator bounds each shard's process at the worker timeout: 480 seconds under the `cold` profile the canonical runner always passes (`run-blaxel-testshards.mjs`, `process.exec` timeout and `process.wait` maxWait). `verify` forwards no `--worker-timeout-seconds`, and neither the runner spec nor the plan proposes to add it. -- **No room to grow.** The tier's hard limit is 75 concurrent shards and `poc` holds 72 (`blaxel-testshards.md:25`), so a private pool cannot reach the standard count. - -The plan half-knows this: "two workers will be slower… and may expose a timeout/capacity limit; inspect the real plan/timings on Blaxel before the gate… report that blocker" (`:86`). The numbers are in the repository now; deferring the decision to the end of slice 3 puts the stall where it costs most. - -**To clear it:** run the final gate on `poc` at the standard 36 shards, from `tv-cleanup-coordinator-rose`, after the private pool's deletion acceptance has run and its sandboxes are gone. That is the converged proposal's arrangement, it is what the proposal's round 3 passed, it keeps Josh's rule (the ban is on the shared host, not the shared pool), and it is the ordinary production condition the new coordinator must work in: shared workers, other branches' runners, legacy locks and known leftovers. The private pool stays for the experiments that contaminate or kill things. If the author has a reason to avoid `poc` for the gate, the plan should state it so it can be weighed; none is visible to me. - -## Refinements (none blocking) - -1. **Whole-surface expansion on the private pool has the same ceiling.** `:19` sends surface expansion through `--pool rose-cleanup`; `--surface` takes the targeted path, one shard on one worker, so `e2e:browser-app` at 376 seconds plus setup is at the limit even on `poc`. For the large surfaces, expand with the planned selection path (`--package` or `--tag` with `--shards 2`) or leave whole-surface coverage to the gate; say which. -2. **`--lock-ttl-seconds` forwarding is a small runner change the plan should name as such.** `:19` says to ensure the canonical CLI forwards it for the driver. The runner spec enumerates no pass-through list, so this is code-governed and needs no spec line, but it is production runner code touched by slice 3 and belongs in that slice's implementation area with a sentence saying ordinary runs are unchanged. -3. **Slice 1 should say where the request inventory comes from.** "Retain a request inventory showing the built page" (`:35`) is the right observation; name the mechanism (Playwright's request events in the existing walk) so it is not read as a new harness. -4. **Record the tier arithmetic once.** Three private sandboxes beside `poc`'s 72 reach the 75-shard limit exactly; another agent's verify during the experiments would contend. The proposal already says to inspect actual usage before provisioning; the plan's execution section should repeat the one line so the implementer does not provision into a full tier. - -## Reviewer checklist - -- **Derivation.** Every slice points at proof anchors and spec sections, not at files alone; the implementation areas are named as areas, with the specs and proofs kept as the requirements. -- **Verification criteria and baselines.** Honest per slice; the final-gate criterion is the blocker above. -- **Placement.** No test, build, lint, type check, browser or server on the shared host; no shared `poc` workers for experiments; isolated-GitHub fixtures untouched. -- **Scope.** Proportionate: no sampler, archive, service or framework; production behavior untouched. -- **Status.** Accurate: no tests or implementation exist; unknowns listed (`:90-92`). - -## Evidence and limits - -- Read the plan in full, the proof and spec diffs since `3fa15d47`, the workflow's planning section, the policy's iteration and verification sections, the runner spec's qualification, verify and selector rules, and the runner's forwarding code; mapped each pending anchor to a slice. -- The timing figures are the baseline's recorded medians at 36 shards; they exclude setup phases and retries, so the two-shard estimate is a floor. -- Not established: the coordinator sandbox's local preflight capabilities for the driver surface; current tier occupancy. - -## Housekeeping - -- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel beyond a script that summed a committed JSON file into the scratchpad. `git fetch origin` updated remote-tracking refs. -- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-plan-review-r2.md b/docs/working/test-harness-resource-cleanup-plan-review-r2.md deleted file mode 100644 index da4e808c..00000000 --- a/docs/working/test-harness-resource-cleanup-plan-review-r2.md +++ /dev/null @@ -1,55 +0,0 @@ -# Plan review, round 2: test-harness resource cleanup (PR #12, 04a6b275) - -**Verdict: PASS.** The final gate is back where it can complete: `npm run verify -- blaxel --pool poc` at the standard 36 shards, invoked from the dedicated coordinator sandbox after the private workers are qualified and deleted, with the private pool kept for every experiment that interrupts, contaminates or deletes anything. The four round 1 refinements are applied. Read as a whole, the plan still has three coherent slices, every pending proof obligation assigned, honest intermediate baselines, scope inside the converged specs, and no execution on the shared development host. Two small refinements follow, neither blocking. - -- Reviewed: the whole plan at `04a6b275`, the diff since `70f27ccb` for the plan, proposal and response, the copied round 1 review (identical to mine), and the runner code the plan's new statements rest on. No spec or proof changed in this revision. -- Branch state: contains `origin/main` (`a8e73fbf`). -- I ran nothing. Every check is a read of files in the checkout or of git history. - -## The blocker is cleared - -| Round 1 asked for | The plan now says | Checked | -| --- | --- | --- | -| A gate that can complete | `npm run verify -- blaxel --pool poc`, standard 36 shards, from `tv-cleanup-coordinator-rose`, after private qualification and deletion of both private workers (`plan.md:82-88`) | Publication-qualifying shape; `verify` forwards `--pool`; the coordinator sandbox is outside pool labels, so Josh's host rule holds. After the two private workers are gone, the tier holds `poc`'s 72 plus the coordinator. | -| A stated reason for leaving `poc`, or a return to it | The arithmetic is recorded (`:90`): 882.5 seconds of file medians, about 441 seconds per worker over two, against the 480-second cold worker timeout the canonical runner does not raise; no timeout control or private capacity is added for this contribution | Matches my own sum of `test/timing-baseline.json`; the response owns the earlier choice as an over-broad reading of the experiment restriction rather than a requirement. | -| `poc` used only for ordinary verification | "No deliberate interruption, contamination, pool deletion or provisioning targets `poc`… Unavailable leases remain an infrastructure failure, not permission to steal a lock or alter the pool" (`:88`) | This is the production condition the new coordinator must work in: shared workers, other branches' runners, legacy locks, known leftovers. The spec's transitional rule keeps other runs safe from it, and it from them. | - -## The refinements are applied - -1. **Large-surface expansion** (`:19`, `:37`): focused files and directly affected helper callers on the private pool; whole large surfaces, `e2e:browser-app` named, wait for the standard-shard gate. The new statement that `--surface` always takes the one-worker targeted path even with `--shards` is true: the targeted branch forces `--shards 1` (`scripts/test/cli.mjs:497`). -2. **`--lock-ttl-seconds` forwarding** is now in slice 3's implementation area as a runner change with ordinary invocations unchanged and no worker-timeout forwarding (`:59`). See refinement 1 below on its stated coverage. -3. **Request inventory** names Playwright request events inside the five existing walks (`:35`). -4. **Tier arithmetic** is in the execution section with the instruction to inspect occupancy before provisioning and to report rather than touch shared workers (`:17`). - -## The whole plan, re-read - -- **Slices.** Unchanged and still right: helpers and the five built-application walks; bounded diagnostics; the lease, dispatch, recovery, scratch, pool and live-qualification lifecycle kept together because it is one safety argument. -- **Obligations.** All 31 pending anchors assigned as in round 1; nothing moved. -- **Baselines.** Slice 1 green for its obligations with the killed-worker backstop assigned to slice 3; slice 2 explicit that the live diagnostics evidence and the lease/download race stay open; slice 3 resolves everything before completion (`:76`). -- **Sequencing on the private pool.** Slices 1 and 2 run focused tests on `rose-cleanup` under the existing coordinator; before slice 3's fresh-provisioning acceptance the idle pool is deleted through the manager so the driver's `^blaxel-t-pool-live` can create it fresh (`:21`); after the driver's deletion case nothing is re-provisioned and the gate moves to `poc`. The order is consistent with `ensure` being fresh-only. -- **Placement.** Edit, commit and push on the shared host; everything else on Blaxel from the coordinator sandbox; the driver as `npm test -- local --surface experiment:blaxel-harness` there; no isolated-GitHub fixtures; no experiment suite as a whole. -- **Evidence.** Red and green commit IDs, commands, run-directory paths and recovered retries retained per slice; the final gate reports the revision actually tested; docs prep deferred to after integrated validation; Josh's spec acceptance and required checks named as merge obligations, not claimed. -- **Limits.** Accurate (`:96`): the implemented tree's gate duration, coordinator capabilities and current tier occupancy are unmeasured and carried into the slices that can measure them. - -## Refinements (none blocking) - -1. **The forwarding's stated coverage overstates what exists.** `:59` says the `--lock-ttl-seconds` forwarding is "covered by the existing CLI delegation contracts". The existing coordinator tests in `test/repo/test-runner-provider.test.ts` and its neighbours read `run-blaxel-testshards.mjs` as source; none asserts the argument list `scripts/test/cli.mjs` hands the coordinator. The live recovery run shows the override arriving but cannot show that an ordinary invocation omits it. Slice 3 should add one small contract: an explicit `--lock-ttl-seconds` appears in the delegated arguments and is absent by default. It is a few lines beside the existing source assertions. -2. **Say what "the tier" counts.** `:17` treats the 75-shard hard limit as a sandbox count (72 + 3). The Blaxel spec phrases it as concurrent shards, and whether idle sandboxes count against it is not established. The sentence already tells the implementer to inspect actual occupancy and limits; adding "whether idle sandboxes count is unknown" keeps the first provisioning attempt from assuming either answer. - -## Reviewer checklist - -- **Derivation.** Slices point at proof anchors and spec sections; implementation areas are named as areas. -- **Verification criteria and baselines.** Honest per slice, and the final criterion is now achievable with the standard runner and pool. -- **Placement.** Nothing on the shared host; `poc` sees only ordinary verification; experiments private. -- **Scope.** Proportionate: no timeout controls, private capacity, sampler, archive, service or framework added; production behavior, retry defaults, shard default and browser version untouched. -- **Status.** Accurate: no tests or implementation exist; nothing has been executed. - -## Evidence and limits - -- Read the full plan, the three working-document diffs, and the runner code for the targeted path and verify pass-through; re-mapped the pending anchors; compared the copied review. -- Not established: current tier occupancy; the coordinator sandbox's local preflight capabilities for the driver surface. - -## Housekeeping - -- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel. `git fetch origin` updated remote-tracking refs. -- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-proof-review-r1.md b/docs/working/test-harness-resource-cleanup-proof-review-r1.md deleted file mode 100644 index 18d973e3..00000000 --- a/docs/working/test-harness-resource-cleanup-proof-review-r1.md +++ /dev/null @@ -1,73 +0,0 @@ -# Proof review, round 1: test-harness resource cleanup (PR #12, 9fd51f49) - -**Verdict: FAIL, on one item.** The proofs cover every promise the touched specs make, at honest shapes, with every mock named and its forfeit placed, and at a cost in proportion to the risk. The spec refinements since `cd8aa453` change wording and resolve the two inconsistencies my spec review named; they change no converged promise. The one blocker: the live assertions and the deliberately failing fixture tests they run have no home. Which suite holds a test is spec authority (the registry's `kind` list and the runner's suite and exclusion rules), and the proofs decide it silently with "an explicit paid-service selection". That is the one thing a proof may not do, and the tests cannot be written until it is settled. The fix is a few sentences in two specs and one line in the proof, or a different framing that needs no spec change at all. Five refinements follow, none blocking. - -- Reviewed: `git diff cd8aa453 9fd51f49 -- specs proofs docs/working`, the full Blaxel, reporting and testing-policy proofs at `9fd51f49`, the proof policy, the testing policy's mocking, composition, shape and cost rules, the reviewer checklist, and the code the hooks name. -- Branch state: contains `origin/main` (`a8e73fbf`); the copied spec review is identical to mine; product and theme specs are unchanged. -- I ran nothing. Every check is a read of files in the checkout or of git history. - -## The spec refinements change no promise - -| Change since `cd8aa453` | Effect | -| --- | --- | -| *reservation* → *lease*/*lock*; *shard attempt* where one shard is meant; `provider attempt` for the directory identity | Wording only. | -| One rule for unreadable or incomplete lock records: reclaim by acquisition or deletion only after a clean worker audit (`blaxel-testshards.md:57`) | Resolves the contradiction my spec review found; the proposal now says the same (`proposal.md:99`). | -| A never-published fresh worker may be deleted without a lease; a worker that cannot answer the ownership check is held (`:63`) | The narrow half of refinement 3, deliberately; the response explains why it declines the wider inference. Consistent. | -| Legacy paragraph marked transitional; two non-requirements dropped; `temporary-cleanup` named as a setup step; sidecars and receipts located under `artifacts/shard-/`; idle-pool deletion stated in operations; setup sidecars finalized before scratch removal; two `Testing` items removed | All from the spec review; none adds or removes a promise. | -| `^worker-build-lifetime` trimmed to the rule | The dropped sentences were derivable detail; see refinement 4 for one proof assertion left pointing at nothing. | - -## Coverage, shape by shape - -**Blaxel spec.** The lifecycle is composed the way the testing policy asks: breadth at two contracts with authored records and recording callbacks (`^blaxel-t-lease-decisions`, `^blaxel-t-lease-order`), the real handoffs crossed once each with real processes, `flock`, procfs and signals (`^blaxel-t-worker-guard`) and a real filesystem (`^blaxel-t-scratch-cleanup`, with the lease and audit precondition declared as the mock), and the spine walked for real through the canonical CLI against Blaxel (`^blaxel-t-retrieval`, `^blaxel-t-recovery-live`, `^blaxel-t-pool-live`), with the temporary-directory directive honored by inspecting the launched browser's descriptors (`^blaxel-t-tmpdir-live`). I mapped every promise in Locking, Abandoned-run recovery, Older runners and pool operations, Temporary files, Pool, Output and Operations to an assertion; the only two without one are small and noted in refinement 5. The forfeits are honest and placed: mocked clock and audit to the guard seam and live recovery; the audit's token filter to live recovery; SDK substitutions to the pool acceptance. The superseded cleanup bullet (`proofs/arch/test-runner/blaxel-testshards.md:67`) correctly retires the TTL assertion that the existing `test/repo/test-runner-provider.test.ts` still encodes; that test changes with the code. - -**Reporting spec.** Bounds, units, unknown-not-zero, counter resets, retention for failed and recovered attempts, attempt history, link validity, credential exclusion and the receipt's contents are all ordered, at the right price: a writer contract with authored readings, one real parent/child/procfs seam that manufactures no OOM, a normalization contract over real sidecar files, and retrieval shared with the provider's live runs rather than a second crash. The hooks (probe reader, clock, output root, write operation, the guarded registry hook for fixture selection) exist or are declared. - -**Testing policy.** `^worker-build-lifetime` gets a filesystem contract with recording builders plus a real Playwright and Vitest worker seam that builds and serves real output, which is the honest crossing the rule needs. `^product-acceptance-built-app` is honored by the design of the five theme and demo-mode walks and ordered no compliance scan, which is right under `^testing-guidance-standing`. - -**Shared execution, runner, themes, artifacts.** Cross-references only, pointing at the provider and reporting assertions; the five converted walks keep their product assertions and change only their loading boundary, which the proof policy allows a proof to do alone (`^proof-change`). All new anchors are unique, every link resolves, cited test files exist, the `TV_TEST_RUNNER_SELFTEST` gate matches its existing use (`scripts/test/cli.mjs:1074`, `planned-shard-worker.mjs:458`), and the proofs index matches the files. - -## Blocker: the live assertions and their fixtures have no home - -`^blaxel-t-retrieval`, `^blaxel-t-recovery-live`, `^blaxel-t-pool-live`, `^blaxel-t-tmpdir-live` and `^reporting-t-resource-live` are automated tests: they spawn the canonical CLI, provision a pool, kill a coordinator, and "a bounded test driver owns and finally cleans up its coordinators and private workers". They need Blaxel credentials, a dedicated pool and minutes of wall clock, and they run "committed native fixture tests" that fail once or always. The coverage model says only that they are "an explicit paid-service selection, not a recursive addition to every broad run" (`proofs/arch/test-runner/blaxel-testshards.md:15`). - -- Which suite holds a surface is spec-owned: `test-registry.md:44` enumerates the kinds in use, `:95` says what `all` and the `experiment` suite contain and why, `test-runner.md:61, 92` name the opt-in suites, and `blaxel-testshards.md:162` lists the suites excluded from Blaxel broad verification. None of them admits a surface that drives Blaxel from inside a test, or fixture surfaces that fail by design. -- The fixtures have the same gap. A planned Blaxel run selects surfaces from the registry on the worker; a fail-once fixture inside `all` would break every broad run, and the reporting proof's registry hook (`TEST_SHARD_CONFIG_MODULE`) reaches a local parent, not a Blaxel worker, unless the coordinator forwards it, which would be a new hook the Blaxel proof does not declare. -- Where they run also matters here: the assignment and Josh's rule keep tests off the shared developer host, and the proposal names a credentialed Blaxel coordinator sandbox. The proof should say that these tests are run from such a host, never from a shared developer host or a Blaxel worker. - -Two ways to clear it, either small: - -1. **Register them.** Add an opt-in surface for the live harness tests and one for the fixture tests (an existing kind such as `experiment`, whose spec already carries one "runs only when selected" class, or a new kind), with the registry, runner and Blaxel specs updated to say so, and the proof's shared live declaration naming the surfaces and the host they run from. Josh sees the spec sentences at PR time with the rest. -2. **Or make them evidence, not tests.** The Blaxel spec's `Testing` section already requires operator-run live evidence for coordinator changes and says what it must exercise. The live scenarios could be ordered the same way: a recorded run from the credentialed coordinator host, with the receipts and run directories as the evidence, and only the fixture surfaces registered. That removes the test driver and its cleanup obligations at the cost of repeatability. - -The proofs should raise this as a finding against the spec rather than choose; I am raising it for them. Everything else in the live assertions stands as written. - -## Refinements (none blocking) - -1. **Grade the five converted walks honestly at the assertion.** `^theme-ac-selection`, `^theme-ac-active-package`, `^theme-delivery-t-active-package-refresh` and the demo-mode shared declaration now describe the built application and cite the existing tests as "covered by", while the coverage models say the adaptation is "(test to be written)". A reader of the assertion alone sees policy-grade coverage of a boundary no test crosses yet. The pool-pagination assertion shows the right form: "(policy-grade test for the existing walk: …; built-application boundary: test to be written)". The theme lines should also name the stable-front proxy, which the coverage model declares but the assertions' "no mocks or test hooks" omits; it is harness transport, not a replaced mechanism, and belongs in the declaration. - -2. **`^blaxel-t-worker-guard` should say where it runs.** "In a private fixture root on a dedicated worker" reads as a live requirement. Everything the seam uses, `flock`, procfs, signals, a filesystem, exists on any Linux CI host, so it should be an ordinary repository test on Linux, like the stale-owner audit's live cases. Say so; otherwise the cheapest real crossing in the proof is priced like an acceptance. - -3. **State the worker-side fixture selection.** Once the blocker settles the surfaces, `^blaxel-t-retrieval` should say the planned run selects them with `--suite selection --surfaces …`, which needs no new hook, so an implementer does not reach for forwarding the registry hook to workers. - -4. **`^testing-t-launch-cleanup` points at nothing.** It cites the policy's "Timing, observable status, cleanup, and flakiness" section, which is two sentences about readiness and sleeps, and the spec sentence it used to derive from was trimmed at the spec review. Under the proof policy that makes it a promise living only in the proof: either fold the per-launch copy into `^testing-t-build-lifetime` as one more owned output under `^worker-build-lifetime`, or drop it. `^testing-t-build-lifetime`'s second pointer to the same section is harmless but adds nothing. - -5. **Two small promises without an assertion.** "Uploads use paths specific to the shard attempt" (`blaxel-testshards.md:43`) and the `temporary-cleanup` setup timing (`:86`) can each be one observation inside `^blaxel-t-retrieval`: the uploaded script's path lies inside the attempt directory, and the shard summary's phases include the cleanup step with its counts. Both cost nothing extra on a run that already happens. - -## Reviewer checklist - -- **Proof coverage.** Every promise at an honest shape; directives honored and the coverage models say where; mocks declared with forfeits; coverage carried elsewhere named. The two gaps are refinement 5. -- **Mocking and boundaries.** No mock sits inside the chain an assertion is about. The audit-filter mock in the guard seam and the lease/audit precondition in the cleanup seam are declared and forfeited to the live runs. Real boundaries (process, filesystem, Blaxel API, browser) are each crossed for real at least once. -- **Placement.** Nothing spec-shaped hides in the proofs except the suite decision in the blocker. -- **Status and citation grades.** New obligations carry "(test to be written)"; the five converted walks need the mixed form (refinement 1). -- **References.** All resolve; the proofs index is regenerated and correct. -- **Cost.** Four live scenarios on a two-worker pool, sharing fixtures; one real nested-runner seam that rebuilds the source CLI once; the rest contracts and cheap seams. Proportionate to a change whose whole risk is reuse before owned processes stop or evidence reaches the caller. - -## Evidence and limits - -- Read every proof hunk with context and the three rewritten proofs in full; checked each new assertion against the spec paragraph it cites and each spec promise against the assertions; verified anchors, links, cited files, the self-test gate and the index by `git grep` and `git show`. -- Not established: whether Blaxel accepts a label change on a `DEPLOYED` sandbox, which `^blaxel-t-pool-live` will show; the exact `experiment` placement rules beyond what `test-registry.md:95` states. - -## Housekeeping - -- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel. `git fetch origin` updated remote-tracking refs. -- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-proof-review-r2.md b/docs/working/test-harness-resource-cleanup-proof-review-r2.md deleted file mode 100644 index ef676d09..00000000 --- a/docs/working/test-harness-resource-cleanup-proof-review-r2.md +++ /dev/null @@ -1,61 +0,0 @@ -# Proof review, round 2: test-harness resource cleanup (PR #12, 3fa15d47) - -**Verdict: PASS.** The placement spec delta gives the live tests and their deliberately failing fixtures a home that fits the registry and runner as they are, with no new kind or suite, and it states Josh's host rule outright. The five proof refinements are applied, one of them better than I asked. What remains is the proof binding the author deliberately held back until this spec review: naming the two surfaces, the coordinator host and the two selection commands in the live declarations. That is a bounded follow-up derived from a paragraph reviewed here, and I do not think it needs another independent round if it stays within what this report describes. Three small refinements follow. - -- Reviewed: `git diff 9fd51f49 3fa15d47` for specs, proofs and working documents; the whole current proof delta against `origin/main` for the Blaxel, reporting, testing-policy, themes, artifacts and bridge proofs; the placement delta against the registry, runner and Blaxel specs and the runner's selection code; the reviewer checklist and the proof policy. -- Branch state: contains `origin/main` (`a8e73fbf`); the copied round 1 review is identical to mine; the spec and proof indexes match the files. -- I ran nothing. Every check is a read of files in the checkout or of git history. - -## The placement spec delta - -| Claim | Where | Checked | -| --- | --- | --- | -| Two `kind: "experiment"` surfaces: `experiment:blaxel-harness` (driver) and `experiment:blaxel-harness-fixtures` (native fixtures, tag `blaxel-harness-fixture`) | `test-registry.md:97` `^blaxel-harness-surfaces` | Uses the existing kind; the sentence that counted the experiment suite's "two placement classes" is removed, so nothing now miscounts. `all` and `e2e-ci` select by kind (`:90-93`), so both surfaces are excluded from broad runs and ordinary CI without a new rule. | -| The driver runs as `npm test -- local --surface experiment:blaxel-harness` on a dedicated credentialed Blaxel coordinator sandbox outside the worker pool, from a committed and pushed checkout, never on a shared developer host or a leased worker | `test-runner.md:147-149` `^blaxel-harness-placement` | States Josh's rule as spec. The local provider accepts every registered suite and surface (`:70`). | -| `--tag blaxel-harness-fixture` dispatches a planned `--suite selection --surfaces …` run; `--surface experiment:blaxel-harness-fixtures` dispatches a targeted run; workers read the committed registry, no forwarded hook | `:151` | Matches the runner today: `--surface/--file/--grep` take the targeted path and `--package/--runner/--tag` become `--suite selection --surfaces` (`scripts/test/cli.mjs:491-510`). The coordinator already accepts `--suite selection` with `--surfaces` (`blaxel-testshards.md:117-118`). A two-shard planned run of one surface needs the existing `--shards 2 --allow-shard-count-override 1`. | -| Deliberate fixture failures stay failed native results; the driver asserts expected outcomes and cleans up its coordinators and pool, reporting what it could not clean | `:151` | Same precedent as the `isolated-github-only` fixtures: registered surfaces that are never in `all`. | -| Blaxel's `Testing` section points at the placement | `blaxel-testshards.md:172` | Reference only. | - -The delta is proportionate: four sentences in the registry, two paragraphs in the runner, one reference. It changes no lock, recovery, cleanup or diagnostic promise. - -## The proof refinements - -1. **Citation grades and the proxy.** Every assertion whose walk is being moved to the built application now carries the mixed form, "policy-grade test for the existing walk: …; built-application boundary: test to be written", and names the stable-front proxy as harness transport: the product theme assertions (`^theme-ac-selection`, `^theme-ac-executable-browser`, `^theme-ac-sandboxed-frames`, `^theme-ac-frame-appearance`, `^theme-ac-active-package`, `^theme-ac-appearance`), the delivery assertions (`^theme-delivery-t-active-package-refresh`, `^theme-delivery-t-frame-browser`, `^theme-delivery-t-frame-appearance-browser`), the demo-mode walks and the bridge's pointer and demo-inert assertions. The author found the frame, pointer and bridge assertions I had not listed; the reset walk and the already-built walks keep plain coverage, correctly. -2. **The guard seam** is now an ordinary repository test on Linux, needing no credentials (`proofs/arch/test-runner/blaxel-testshards.md:78`). -3. **The reporting hook** says it serves the local parent/child seam only and is not forwarded to workers (`reporting.md:22`). The worker-side selection commands are now in the spec and await the binding below. -4. **Launch cleanup** is folded into the real worker seam (`testing-policy.md:25`) and the standalone assertion is gone. The response is right that my premise was too narrow: the policy's cleanup section does say that cleanup of long-running modules is core to correctness, below its motion subsection. The consolidation still stands on its own merits, one fixture and one launch sequence fewer. -5. **Upload paths and the `temporary-cleanup` phase** are observed inside `^blaxel-t-retrieval` on the run that already happens. - -## What is left: the binding - -The Blaxel proof's coverage model still says the live tests are "an explicit paid-service selection" (`:15`), and the shared live declaration (`:84`) and `^reporting-t-resource-live` still name no surface, host or selection. The response records this as deliberately pending spec convergence and lists what it will add. For that follow-up to stay bounded, it should contain only: - -- in the shared live declaration: the driver surface, the fixture surface, the coordinator host rule, and the two selection commands, with links to `^blaxel-harness-placement` and `^blaxel-harness-surfaces`; -- the coverage-model sentence replaced by one that says where the live tests live and that they are excluded from `all` and ordinary CI by kind; -- one contract in the registry or runner proof over the committed registry: `all` and `e2e-ci` resolve without either surface, the tag resolves to the fixture surface alone, and the driver's dispatched selections name only the fixture surface. That is a source-level contract, not a live run. - -Within that scope I would not ask for another round. Anything beyond it, the owner judges under the bounded-refinement rule. - -## Refinements (none blocking) - -1. **The runner's `Selectors` paragraph should say what the new section relies on.** `test-runner.md:70` says narrower remote work uses "surface, file, or grep selection through the target path" and is silent on `--package`, `--runner` and `--tag`, which the code sends to the planned selection path. The placement paragraph depends on that path. One clause at `:70` makes the spec self-consistent instead of leaving the only statement of the planned-selection route inside a specialized section. -2. **`test-runner.md:92` now has a third opt-in placement it does not mention.** "The `telemetry-posthog-roundtrip` and `daemon-acceptance` suites are not broad verification" could add "and the Blaxel harness surfaces ([#Blaxel harness qualification](#Blaxel harness qualification))" so a reader of the guardrail section sees all three. -3. **Minimum shard shape for the planned fixture run.** The fixture surface must hold at least two files for `^blaxel-t-retrieval`'s two-shard run; worth one clause in the fixture surface's description so the first implementer does not discover it from a plan failure. - -## Reviewer checklist - -- **Authority and placement.** Surface declarations in the registry spec, execution placement in the runner spec, a reference from the provider spec. Nothing spec-shaped remains in the proofs once the binding lands. -- **Proof coverage, mocking and boundaries.** Unchanged from round 1 and still sound; the consolidated worker seam keeps its real crossing. -- **Citation grades.** Honest everywhere I looked. -- **References.** All new anchors resolve; indexes regenerated. -- **Size and simplicity.** No new kind, suite, hook or framework; the driver reuses canonical commands. - -## Evidence and limits - -- Read the full spec and proof diffs, the response's round 1 section, and the current Blaxel, reporting and testing-policy proofs; checked the placement claims against `cli.mjs` and the coordinator option table; confirmed the policy's cleanup sentence the response cites. -- Not established: whether a `local` run of the driver surface on the coordinator sandbox passes local preflight's capability checks as that sandbox is provisioned, which the first run will show. - -## Housekeeping - -- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel. `git fetch origin` updated remote-tracking refs. -- Primitive produced no output during this review, so I have nothing to report on its usefulness. The response records a Primitive decision (`dec_7893f958`) from the author's session; I have not seen it and do not assess it. diff --git a/docs/working/test-harness-resource-cleanup-review-r1.md b/docs/working/test-harness-resource-cleanup-review-r1.md deleted file mode 100644 index 5c3043b7..00000000 --- a/docs/working/test-harness-resource-cleanup-review-r1.md +++ /dev/null @@ -1,131 +0,0 @@ -# Proposal review, round 1: test-harness resource cleanup (draft PR #12) - -**Verdict: FAIL.** Two blockers, both about scope and consequences, not about the diagnosis. The diagnosis is supported, the three recommended fixes are the right ones, and the core `/tmp` answer is safe. The proposal then adds a rework of the worker lease that the cleanup does not need, and it leaves out what happens while other branches keep running the old harness on the same pool. - -- Reviewed: `docs/working/test-harness-resource-cleanup-proposal.md` at `1652787e` on `thopter/test-harness-resource-cleanup`, with the investigation report and both evidence folders. -- Against: Josh's request (the investigator's recommended fixes, and whether `/tmp` on Blaxel workers can be cleaned safely before or after runs), the contribution workflow, the reviewer checklist, and the code and specs the proposal cites. -- I ran nothing. Every check below is a read of files in the checkout. - -## Is the diagnosis supported? Yes - -I checked the report's claims against the retained evidence and the code. - -| Claim | Evidence | Result | -| --- | --- | --- | -| Renderers were killed for lack of memory | Kernel logs: two kills in the shard replay, three in the isolated case, each a `chrome-headless` process | Proven | -| The unbundled page's response buffers are the consumer | The forced-collection experiment released 220 buffers of 2 MiB in each of two renderers; a passing run fell to about 0.55 GB of free memory | Proven for retention; the failing allocation call is inferred, and the report says so | -| The worker's writable filesystem is RAM | Mount table: overlay with its upper layer under `/mnt/tmp`; idle shared memory tracks used root space on both inventoried workers | Supported | -| `getSourceCLIEntry()` leaks one directory per worker process | `test/helpers/product-server.ts:67-76`: the cache is set at line 70, before the build at 71; nothing removes the directory | Confirmed | -| `buildVersionedWebBundle()` leaks one directory per version | `test/helpers/versioned-web-bundle.ts:39-41, 62-70`: no removal, and a failed build stays cached | Confirmed | -| 267 and 146 leaked directories on `poc-131`; 17 and 27 on `poc-03` | Inventory files | Counts and byte totals match | -| The lock is reclaimed on time alone, and released before results are downloaded | `scripts/run-blaxel-testshards.mjs:406` (expiry), `:859` (exit trap), `:457-502` (download after the process ends) | Confirmed | -| The 17 web test files in the conversion table are the `ProductServer.appURL()` users | A search finds exactly those 17, plus the three desktop files the proposal excludes | Confirmed | -| The product runtime lacks the `url-unsupported` view | `test/helpers/product-server.ts:103-132` copies the index, assets, two views and canonical only | Confirmed | - -The report is careful about what it does not know, and the proposal carries those limits forward. - -## Blocker 1: the lease rework is not needed for safe cleanup, and its consequences are not examined - -The proposal says cleanup "therefore extends the existing lease boundary" and calls this "a narrowly required correction". It asks for all of the following: -- the coordinator holds the lease through download and disposal; -- the worker stops unlocking at shell exit; -- reclamation checks the prior supervisor's process identity instead of the expiry time; -- acquisition stays exclusive under racing coordinators; -- an ambiguous owner keeps the worker unavailable; -- results move to an attempt-specific directory. - -**Safe cleanup does not depend on any of that.** The proposal's own directory design already makes it safe: -- A job creates its attempt directory under a random name, so it owns it by construction. It can remove it in the exit path that exists today, before that path releases the lock (`run-blaxel-testshards.mjs:859`). -- Reclaiming an older attempt directory needs one local check: is the recorded supervisor, by process ID and start time, still alive? If it is, leave the directory. That check protects a live job even when the lock has wrongly changed hands. -- Results do not live in the scratch directory, so the download race does not bear on scratch cleanup. - -**The lease weaknesses are real but separate.** -- The download race is a genuine defect: the lock is released at shell exit, and the next job's first step deletes the results directory. -- The expiry concern is weaker than the text suggests. The lock lives 900 or 1200 seconds against a worker timeout of 240 or 480 seconds (`run-blaxel-testshards.mjs:1176-1178`). The Blaxel spec names expiry deliberately as "the final bound" for a worker whose cleanup could not be confirmed (`specs/arch/test-runner/blaxel-testshards.md:29-31`). -- The proposal itself says no observed failure is attributed to these races. - -**The consequences are not handled.** -- *Availability.* Today a worker with unconfirmed cleanup comes back when its lock expires. Under the proposal an ambiguous owner keeps it unavailable. On a pool that every agent shares, that turns a twenty-minute delay into a worker that stays out until someone intervenes. The proposal does not say who clears it or how. -- *Mixed versions.* Each coordinator runs from its own branch and uploads its own shard script. Branches that have not taken this change will keep reclaiming on expiry and unlocking at exit. The new guarantees do not hold while they exist, and the proposal mentions this only for the one-time cleanup. -- *Spec weight.* It replaces a rule the Blaxel spec states on purpose. That is a decision for Josh on its own merits, not a side effect of cleaning `/tmp`. - -**To clear this:** take the lease rework out of this contribution and record the download race as its own item; or keep it, but present it to Josh as a separate decision with the consequences above worked through, and drop the claim that cleanup requires it. - -## Blocker 2: the proposal does not say what happens while other branches still run the old harness - -The helpers and the shard script are per-branch. A run dispatched from a branch without this change still writes `television-source-cli-e2e-*` and `television-web-bundle-*` straight into `/tmp` on the shared workers, and nothing in the new routine cleanup removes them: it deliberately does not infer ownership from a prefix. - -So three statements in the proposal are true only for updated branches: -- "A worker should start its next shard without accumulating the preceding shard's disposable files." -- The legacy removal is a "one-time operation". -- "The normal before/after cleanup covers future runs regardless of which helper created their temporary files." - -Old-branch runs will keep leaking, slowly (about 4 MB per leaked runtime), until every active branch has taken main. That may be perfectly acceptable. Josh should see it and choose, because the alternatives differ in cost: -- accept the residue and repeat the legacy cleanup once branches have caught up; -- let the routine pre-run step remove these two known generated prefixes on a leased idle worker; -- recreate the pool at a chosen time. - -**To clear this:** state the mixed-version period, its effect on each claim above, and the option the proposal recommends. - -## Is the `/tmp` answer safe and evidenced? Yes at its core - -- **"Not blanket" is right and evidenced.** The inventories show the lock, the uploaded shard scripts, the provider's upload directory and the X11 directory all in `/tmp`. Deleting everything before a run would remove the job's own script and lock. -- **"Yes for files the runner owns" is right.** A per-attempt directory with `TMPDIR` pointing into it gives cleanup an exact boundary. It also catches leaks nobody has found yet. The inventories show more leaked prefixes than the two helpers: 222 uploaded shard files, 33 `tv-electron-env-*`, and dozens of `tv-attest-*` directories from test fixtures. -- **The wiring claims hold where I could check them.** Both helpers use `os.tmpdir()`. The Playwright and Chromium claims match my knowledge of those sources, but I did not fetch them. The proposal correctly says a live run must still observe Node paths and Chromium's shared-memory files under the attempt directory. -- **The stated limits are the right ones:** hard-coded `/tmp` paths and the X11 socket directory do not move; deleting a directory does not free memory that a live process still holds. - -The answer would be easier for Josh to act on if it came first and plainly: -- **After a run:** yes, the run removes its own scratch before releasing the worker. -- **Before a run:** yes, for scratch whose recorded owner is dead; never everything. -- **What is already there:** a separate decision (see refinement 2). - -## Is the scope proportionate? Partly - -Apart from the lease rework, the three fixes are the investigator's three recommendations. Their weight in the proposal does not match their effect, and the proposal should say so. - -## Refinements (non-blocking) - -1. **State how much each fix buys.** - - On `poc-131`, `/tmp` held 1.2 GB of 4.3 GB used on a 6.3 GB root. The rest is the checkout and `node_modules` (1.4 GB), the Playwright browsers (0.9 GB) and caches. On `poc-03`, `/tmp` held 0.16 GB of about 3.0 GB. - - The theme-reset test consumed roughly 3.2 GB by itself in a passing run. - - So even a perfectly clean worker runs that unbundled test within about half a gigabyte of the limit. The built-application change removes gigabytes per test; cleanup recovers at most about one, on the worst worker seen. - - Both are worth doing, but the problem statement should give these sizes so the cleanup machinery is sized to its benefit. - -2. **Weigh recreating the pool against the bespoke legacy cleanup.** - - The Blaxel spec already defines provisioning as delete-then-ensure and says it is fresh-only (`blaxel-testshards.md:21-23`). Recreating the pool once clears every worker with no new code. - - The proposal offers that only as a fallback. It prefers a "drained worker under an exclusive maintenance lease" with layout checks, and does not say whether that is new code or an operator procedure. - - Say which, and why it beats recreation. - -3. **Name the minimum diagnostics that would have identified this incident.** - - Free memory, shared memory, free bytes on the temporary filesystem and the kernel's out-of-memory kill counter, read at surface start and end, plus the kernel's kill lines on failure, would have been enough. - - The periodic sampler, the per-process descriptor summaries, the attempt records and the recording-time caps are the investigation's tools. - - The diagnostics section reaches into five specs. Mark the minimum as the commitment and the rest as optional. - -4. **Two validation questions need answers before work starts.** - - *Which pool?* Lease, cleanup and interruption experiments must not run on the shared `poc` pool that other agents use. The proposal says "an isolated Blaxel worker" without naming a pool or its cost against the 75-shard limit. - - *Which host runs the final gate?* The proposal requires `npm run verify -- blaxel` from "a dedicated Blaxel coordinator host". Nothing in the documents shows such a host exists. The workflow accepts ready-PR GitHub CI as full validation, and the Blaxel spec asks only for a live targeted run and a planned-shard run for coordinator changes. Either name the host or use that route. - -5. **Say what stays exposed.** 41 test files load the unbundled page. The proposal converts the 17 that go through `ProductServer.appURL()` and leaves the three desktop ones. About 21 others reach the development page without the product server and keep the same per-load cost. Say whether any of them reload or open several pages, since that is what made theme-reset fail. - -6. **Give the reason for fixing the helpers as well as adding scratch space.** On Blaxel, attempt scratch removed at shard end would clear the helpers' directories without touching the helpers. The helper fix is what protects hosts with no shard script. This development host has 1,388 `television-source-cli-e2e-*` and 727 `television-web-bundle-*` directories in `/tmp` today. That is the argument for doing both, and the proposal does not make it. Whether to clean this host is the owner's call. - -7. **Sequencing with PR #9.** Both change `theme-product-acceptance.test.ts`, and both touch `test/helpers/stable-front-proxy.ts`: PR #9 adds a hook, and this proposal reuses the proxy for restart walks. Say which lands first. - -## Reviewer checklist - -The proposal changes no spec yet, so most checklist items apply at the next stage. Two apply now: - -- **Authority.** The derivation section names the owning specs correctly. With the lease rework removed, `preflight.md` and most of the `blaxel-testshards.md` changes fall away. -- **Testing directives.** The proposed policy line, built application by default for product browser acceptance, is a real human-owned directive and belongs in `testing-policy.md`. It should come with refinement 5's statement of what remains on the development page and why. - -## What I could not establish - -- Whether Blaxel kills a shard process at its timeout. That decides how real the expiry overlap is; I assumed it can fail. -- The make-up of the 0.9 GB of Playwright browsers on the workers, or whether any of the non-`/tmp` baseline is reclaimable. -- The Chromium and Playwright source claims, which I did not fetch. -- Why `poc-32` and `poc-15` started with less free memory than the passing workers. Neither was inventoried, so the share due to `/tmp` there is unknown. - -## Housekeeping - -- Read-only: no edits, commits or pushes, and nothing run on this host or on Blaxel. I listed names in this host's `/tmp` to count the leaked directories; nothing was changed. -- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-review-r2.md b/docs/working/test-harness-resource-cleanup-review-r2.md deleted file mode 100644 index a5550396..00000000 --- a/docs/working/test-harness-resource-cleanup-review-r2.md +++ /dev/null @@ -1,72 +0,0 @@ -# Proposal review, round 2: test-harness resource cleanup (draft PR #12, d3570bf4) - -**Verdict: FAIL, on one blocker.** The proposal now follows Josh's decisions, its evidence still holds, and its scope is much closer to its benefit. What remains is a hole in the lock design it was asked to make correct: it says how a worker becomes locked but not how a worker locked by a dead coordinator ever becomes usable again. Four refinements follow, one of them a decision for Josh. - -- Reviewed: `docs/working/test-harness-resource-cleanup-proposal.md`, the review response and the unchanged investigation report at `d3570bf4`, against the reviewer checklist, Josh's decisions as the response records them, the code at the branch base `b5477211`, and current `origin/main` (`a8e73fbf`), including PR #32. -- I ran nothing. Every check is a read of files in the checkout or of main's history. - -## What the revision gets right - -- **Round 1's blockers are resolved as Josh decided.** The lock work is now Josh's requested correction, not a redesign presented as a prerequisite. Old branches are handled by removing the two known prefixes on every run under the lock, which I checked against the old runner: it reclaims a lock only when `expiresAt` is positive and past (`scripts/run-blaxel-testshards.mjs:406`), so a new lock written with `expiresAt: 0` is left alone, as the proposal says. -- **The relative benefit is stated** (round 1 refinement 1): the historical 3.2 GB test cost against 1.16 GB and 0.079 GB of leaked prefixes on the two inspected workers, with the other 2 to 3 GB of baseline named as out of scope. -- **Diagnostics are reduced to the minimum** that would have identified the incident, with the sampler and descriptor inventories made optional (refinement 3). -- **The infrastructure is named**: a private two-worker pool for experiments and a coordinator sandbox for the final gate, never the shared `poc` pool for experiments (refinement 4). -- **Remaining exposure is listed** with per-file behaviour (refinement 5). I spot-checked four of those claims at `b5477211` and all hold: the telemetry walk's two tabs and reload, the artifact-navigation `beforeEach` reload, the silent-reconnect reload in `auth.02`, and the reconnect test's count of top-level navigations. -- **The PR #32 assessment is honest.** It stops claiming the conversions repair the measured leak, and it requires any resource comparison to use a baseline containing #32. -- **Factual claims check out**: PR #9 is `f81a2465` on main; the pool script deletes through the provider API (`scripts/manage-blaxel-testshards.mjs:76`) and inspects through `process.exec` (lines 94, 115); the download race is unchanged on main, whose coordinator diff since the base only moves the checkout and dependency scripts into a module. - -## Blocker: recovery of a locked worker is not defined - -Josh asked for a lock that is correct for our own runner and pool scripts. The proposal removes time-based reclamation and holds the lock through teardown and download. That is right as far as it goes. It then says (lines 70 and 72): - -- "A coordinator crash can leave a lock indefinitely." -- "Once the original coordinator is confirmed ended, recovery reuses the same stop/audit, download, and matching-lock release steps." -- "If execution status, process teardown, or retrieval cannot be confirmed, leave the lock held and report the specific reason. No timer or blind `rm -rf` is a recovery procedure." -- An expired legacy lock "is not reclaimed merely because of its age; uncertain ownership needs the same recovery checks." - -What is missing: - -1. **No entry point.** Nothing says what runs recovery: a runner command, a pool-script command, or a person following a procedure. Spec derivation would have to invent it. -2. **"Confirmed ended" has no method.** The lock records the coordinator's host and PID (`lockCommand`, line 396). A recovery run on the same host can check that PID; most coordinators do run on this host, so that covers most crashes. From any other host there is no way to confirm it. The proposal should say which evidence suffices: a dead coordinator PID on the same host, or an operator's assertion otherwise. -3. **Undownloaded results of a dead coordinator are never released.** The results exist to be downloaded by the coordinator that asked for them. If it is gone, recovery can retrieve them to a run directory nobody will read, or discard them. Either is fine; the proposal has to choose, because "until results have been downloaded" otherwise means forever. -4. **The cost is stated but not sized.** Every coordinator killed without its signal handler running (`kill -9`, a host reboot, a lost session) removes a worker from the shared pool until someone runs recovery. With many agents sharing one pool, that is a steady drain. The old design healed this in twenty minutes; the new one heals it never, by itself. - -A correct and simple design within Josh's calibration is available and worth putting to him as the decision it is: reclaim a held lock automatically when worker-local evidence shows the run is over (the recorded shard supervisor is dead by PID and start time, the owner audit finds no live or stale owners, and results are either marked downloaded or older than a generous grace period), and otherwise hold and report. That uses only our own records, keeps the pool self-healing, and loses nothing but the reports of a coordinator that died before reading them. If Josh prefers manual recovery, the proposal can say so, with the entry point and evidence above. - -## Refinements (none blocking) - -1. **The built-application conversion needs a decision from Josh, with the post-#32 picture in front of him.** - - PR #32 removes the mechanism the investigation measured: Vite services no longer answer `304`, so a reload no longer holds about 465 MB. The proposal says so and keeps the 17-file conversion on two other grounds, product fidelity and fewer requests, while admitting their benefit is unmeasured. - - Main has moved the other way for in-process servers. Twenty-one test files and a shared helper now reach the application through the new `appURLForServer()` function, which puts a test's own server behind the Vite proxy on purpose (`test/helpers/product-server.ts`, main). The proposed testing-policy directive, built application by default for product browser acceptance, would read as contradicting that unless it is scoped to `ProductServer` walks. - - Josh said to keep "suitable" built-app acceptances. The proposal should offer him the choice plainly: the full set on the fidelity argument, or the walks whose proofs name the built application or that reload several pages, with the rest left to a later contribution. Either way, scope the directive. - - The conversion table is a base-revision inventory. On main, `artifact-isolation.test.ts` and `artifact-move.test.ts` in the web suite and `artifact-isolation.test.ts` in the desktop suite also use `appURL()`. The proposal defers reconciliation to implementation, which is acceptable, but it inspected main at `a8e73fbf` and could name them now. - -2. **The runner's own scratch is treated more cautiously than the legacy prefixes.** Legacy `television-*` directories are removed on the strength of the lock and the idle check. The runner's own `/tmp/tv-testshards/` directories, which are more certainly ours, additionally need intact metadata and "confirmed completed/recovered ownership", and "missing or inconsistent metadata needs recovery". A crashed run with half-written metadata therefore leaves its scratch until the manual recovery above. Josh's decision covers "stale per-run scratch". Under the lock, after the owner audit confirms no live or stale owners, remove every attempt directory except the current one. That is what "no adversarial design" allows. - -3. **Two statements about old coordinators ask for more than is needed.** - - "Carry the small lock lifecycle correction into branches still dispatching to the shared pool" and the final verify's precondition that "active coordinators carry the lock correction". Branches take the correction when they take main; nothing can make that happen sooner, and nothing requires it. A new run's non-expiring lock already keeps old coordinators off its worker. The only effect of an old coordinator on a new run is that its workers may show expired legacy locks the new runner will not reclaim, which costs capacity, not correctness, and is another reason to settle the blocker. - - "After old in-flight runs have finished" as a precondition for the final run cannot be known on a shared pool. Drop it. - -4. **Two smaller points.** - - The investigation says the workers run Chromium 148 through the coordinator's build alias; PR #32's spec attributes the `304` behaviour to Chromium 145 and says other versions may differ. Both documents are in the tree. Say which version the workers actually run, or that it is unknown, so the proofs measure the right thing. Browser-version alignment itself can stay out of scope. - - The final gate runs `npm run verify -- blaxel` from a coordinator sandbox inside Blaxel. That sandbox needs the Blaxel credentials and the GitHub token the runner reads; the proposal should say how they get there without being persisted. - -## Reviewer checklist - -The proposal changes no spec yet. On the items that apply now: - -- **Authority.** The derivation section names the right owners, and now says the Blaxel spec's TTL escape and shell-exit unlock are replaced rather than left in conflict. Good. -- **Testing directives.** The one proposed directive is the place where refinement 1's scoping belongs. -- **Writing.** The document is long for a proposal, but the length is in the lock and cleanup sections that Josh asked to be decided here. The remaining-exposure table is the right shape: per file, what was seen and why it is deferred. -- **Status.** The review-status section is accurate and does not claim convergence. - -## Evidence and limits - -- Read the proposal, the response and the investigation in full; diffed the copied round 1 review against mine (identical) and the investigation against its round 1 version (unchanged). -- Read the lock, trap and timeout code at the base and main's coordinator diff; the pool script; main's `product-server.ts` and `playwright.ts` changes; PR #32's commit, spec and proof text; and the four spot-checked tests at `b5477211`. -- Not established: whether Blaxel kills a shard process at its timeout; which Chromium build the workers run today; main's current count of development-page loaders beyond the `appURL()` and `appURLForServer()` lists. - -## Housekeeping - -- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel. `git fetch origin` updated remote-tracking refs. -- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-review-r3.md b/docs/working/test-harness-resource-cleanup-review-r3.md deleted file mode 100644 index 83a67eee..00000000 --- a/docs/working/test-harness-resource-cleanup-review-r3.md +++ /dev/null @@ -1,70 +0,0 @@ -# Proposal review, round 3: test-harness resource cleanup (draft PR #12, 57a701d0) - -**Verdict: PASS.** Round 2's blocker is resolved: the proposal now says what recovers a worker whose coordinator has gone, on what evidence, what happens to its reports, and what it costs, and the design follows Josh's decisions. The recovery design is close to as simple as it can safely be. Four refinements follow; the first two are worth doing before spec derivation, and none changes the verdict. - -- Reviewed: `docs/working/test-harness-resource-cleanup-proposal.md`, the review response and the copied round 2 review (identical to mine) at `57a701d0`, against Josh's round 2 decisions as the assignment and the response record them, the reviewer checklist, the code at the branch base `b5477211`, and current `origin/main` (`a8e73fbf`). -- I ran nothing. Every check is a read of files in the checkout or of git history. - -## The blocker is resolved - -| Round 2 asked for | The proposal now says | Checked | -| --- | --- | --- | -| An entry point | The canonical runner's `acquireAny()` recovers on finding an expired `recoverAfter`; the pool script's mutations reuse the same helper. | Consistent with `acquireAny` at `scripts/run-blaxel-testshards.mjs:333-358` (base), the only acquisition path. | -| Evidence that the coordinator is gone | The coordinator renews once a minute; the deadline is worker-local time of the last renewal plus the configured worker timeout plus 15 minutes; renewal cannot revive an expired reservation; only the coordinator renews. | Worker-local time avoids clock skew. Coordinator-only renewal is right: a shard that renewed for itself could hold a worker forever after its coordinator died. | -| A decision on undownloaded reports | Discarded when recovery runs, recorded in a receipt with the incoming attempt's setup sidecars; loss never becomes a passing result. | Resolves the "forever" reading of download-before-release. | -| The cost | 19 or 23 minutes after the last renewal, then one acquisition and bounded cleanup; quiet and active leftovers wait the same interval. | Sized and stated. | - -Code claims the design rests on, all verified: - -- A new lock written with `expiresAt: 0` is left alone by the old runner, which reclaims only a positive past timestamp (`run-blaxel-testshards.mjs:405-406`, base). -- An old shard's exit trap releases only a lock carrying its own `lockId` (`release_lock`, lines 773-775), so an old shard ending cannot remove a new owner's lock. The proposal could say this; it is why the legacy case is a skip, not a hazard (refinement 3). -- The owner token lives in one variable, `TV_TEST_SURFACE_OWNER`, and a surface supervisor replaces it for its children (`scripts/test/surface-supervisor.mjs:62`, main). A second attempt-level token that nested supervisors do not replace is therefore needed, as the proposal says. -- The existing reaper leaves processes with live supervisors untouched (`scripts/test/stale-owner-reaper.mjs:24, 96`). Recovery has to stop the abandoned run's live supervisor as well, which the proposal states without weakening the reaper for other owners. -- The worker timeout is the bound on the shard process itself: `process.exec({ timeout })` and `process.wait({ maxWait })` at lines 448 and 457. The spec's TTL and "final bound" sentences the proposal replaces are at `specs/arch/test-runner/blaxel-testshards.md:27-31`. - -## Is the recovery design as simple as it can safely be? Nearly - -What is simple and right: - -- One sequence for a quiet worker and for a still-active run, with no second liveness test and no report-age clock. That is the shape Josh asked for. -- One renewal, from the coordinator only, with the reason given. -- One guard (`flock` on a fixed file) for all worker-local lock changes. This is justified by a race in our own code: a renewal that read the lock before recovery replaced it would otherwise overwrite the new owner's record and fail the new run. It is one line of shell and is released by the kernel when a process dies. -- Orphaned reports are discarded, not archived; recovery is repeatable if it dies midway; a coordinator that finds its identity gone stops and reports. - -What can go without losing safety is in refinements 1 and 3. The one gap in the simple story is refinement 2. - -## Refinements (none blocking) - -1. **Use one fixed recovery interval, not "worker timeout plus 15 minutes".** Renewal is a timer, so the shard's execution bound has no bearing on how long a healthy coordinator can go without renewing: it renews every minute whatever the timeout. The timeout term adds a field recorded at acquisition, a custom-timeout case in the contract coverage, and two derived numbers to keep in step with the profiles. One interval does the same job. Reuse the existing `--lock-ttl-seconds` option (`blaxel-testshards.md:76`) as its override, which also lets the live recovery check on the private workers use a short interval instead of waiting at least 15 minutes per case. The renewal cadence is a free choice too: once a minute is 36 API calls a minute for a full run; a longer cadence with the same deadline costs nothing in safety. - -2. **The pool script must be able to delete the worker that needs repair.** The proposal has `delete` acquire each worker's lock and refuse busy ones, and names "operator repair" as the way out when a process cannot be stopped. Those two rules meet badly: the worker that cannot be cleaned is exactly the one the script would refuse to delete. Deleting a sandbox is the cleanup, so `delete` should refuse only a reservation that is still being renewed and proceed on one past its deadline, including one held because cleanup failed. Say also that a deleted worker is not re-added until the pool is recreated, since `ensure` is fresh-only (`scripts/manage-blaxel-testshards.mjs:41-47`); that is the status quo, and one sentence keeps an operator from expecting otherwise. - -3. **An expired legacy lock with live work is a skip, not a repair.** The proposal says to "keep and report" such a lock and, in its status section, that "unidentifiable legacy work still needs repair". Because the old shard's trap removes its own lock when it ends, an updated runner only has to skip that worker in this acquisition, as it already does for an unexpired lock; the worker frees itself. The only stuck case is an old shard that never exits, which is the status quo and ends when branches take main. Dropping the repair language removes a case from the design and from the spec. - -4. **Three sentences to trim or fix.** - - The 128 KiB cap on the recovery receipt. A receipt is a few hundred bytes of identities and outcomes, and the attempt's 8 MiB diagnostic budget already bounds it. - - "Fresh provisioning runs with dispatch to that pool stopped until setup and architecture verification finish." Nothing stops dispatch to a shared pool. If the intent is that `ensure` holds each new worker's lock until provisioning is verified, say so; otherwise rely on the architecture check that already skips an unprovisioned worker (`verifySandboxArchitecture`, lines 360-380, base) and drop the sentence. - - "Manual pool operations use the same drained-pool discipline." No such discipline is defined now that recreation is out of scope. - -## The rest of the proposal - -- **Built-application conversion.** Five cases in two files is a defensible judgment on the post-#32 evidence, and the proposal's facts hold on main: both files use a `ProductServer` through `appURL()` (`browser-demo-mode.test.ts:75-79`; `theme-product-acceptance.test.ts:694-695, 1065-1068`); the two theme cases restart the server (lines 845, 1785), and `restart()` boots a new port-0 backend (`test/helpers/product-server.ts:26-32`), so the stable front proxy is required for them, as the proposal says; `views/url-unsupported` is built (`packages/web/vite.config.ts:178`), served from Vite under `appURL()` (`shouldProxyToVite`, `product-server.ts:393-400`), and not copied by `installProductRuntimeAssets()` (lines 103-132), so the asset copy is a real prerequisite. The product proof calls the persistence walk "production shell modules through the test Vite server" at line 42 and the same walks "built application" at lines 44 and 51; the delivery proof does the same at lines 61 and 74-75. That is the description change the proposal anticipates. The testing directive is now scoped to `ProductServer` acceptances and excludes `appURLForServer()` walks, which settles round 2's refinement 1. -- **Round 2 refinements 2 to 4** are applied as proposed: every non-current attempt directory is removed after a clean audit; the old-branch preconditions are gone; the Chromium 148/145 discrepancy is stated and the build alias is still on main (`run-blaxel-testshards.mjs:923`); the credential path is supported by the code: the SDK reads `BL_API_KEY` and `BL_WORKSPACE` from the environment (`@blaxel/core` `common/settings.js:16-24`, `authentication/index.js:9-11`), `readBlaxelGithubToken()` reads the file and then `BLAXEL_TV_GH_TOKEN` (`scripts/test/blaxel-github-token.mjs:7-18`), and the runner passes only `GH_TOKEN` to workers (line 448). -- **Exposure table.** The three `appURL()` users found in round 2 are now listed; the reload in web `artifact-isolation.test.ts:149` and the second page in `artifact-move.test.ts:54` are as described. - -## Reviewer checklist - -- **Authority.** The derivation list names the right owners and says the Blaxel spec's TTL escape and shell-exit unlock are replaced. Add the fate of `--lock-ttl-seconds` (refinement 1) so the option table is not left describing a removed mechanism. -- **Testing directives.** The one directive is scoped and phrased as a default, not a prohibition. -- **Proportionality.** The lock and cleanup sections are long because Josh asked for those decisions here; the diagnostics commitment is the minimum; the conversion is two files. The length is in proportion to what is decided. -- **Status.** Accurate: ready for re-review, not converged; no spec, test or code edited; nothing run. - -## Evidence and limits - -- Read the proposal, the response and the copied review in full and diffed them against `d3570bf4`; read the base coordinator's lock, dispatch, wait, trap and timeout code, main's reaper, supervisor, token reader and pool script, the SDK's credential code, the two conversion targets, the three exposure-table files, both theme proofs, and the Blaxel spec's locking section. -- Not established: whether Blaxel's process `timeout` ends descendants (the proposal assumes not, which is the safe side); any rate limit on renewal calls; whether the provider can deliver a dispatch after a recovery interval (the shard-start lock check is cheap either way). - -## Housekeeping - -- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel. `git fetch origin` updated remote-tracking refs. -- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-review-response.md b/docs/working/test-harness-resource-cleanup-review-response.md deleted file mode 100644 index 08919443..00000000 --- a/docs/working/test-harness-resource-cleanup-review-response.md +++ /dev/null @@ -1,356 +0,0 @@ -# Resource-cleanup proposal: review response - -Proposal round 3 at `57a701d0`, spec round 1 at `cd8aa453`, proof round 2 at `3fa15d47`, and plan round 2 at `04a6b275` all **PASS**, relayed by the supervisor. Proof round 2 also approves the narrow placement spec delta. All three implementation slices have converged; the integrated gate passed and independent integrated review is next. Earlier sections retain the response history, including the superseded private full-gate arrangement. Reviewer: Claude Fable (Nina), arranged by the supervisor. - -Independent round 1 reviewed `1652787e`. The [full review](test-harness-resource-cleanup-review-r1.md) remains unchanged: FAIL on two blockers. The response committed at `99b7ed93` narrowed the work pending Josh's decisions. The following sections through round 2 record that history, not the current review verdict. - -## Josh's scope decisions - -All of the following belong in PR #12: - -- Keep a worker exclusively reserved through process cleanup and result download. Account for our pool script's direct Blaxel API calls as well as shard coordinators. Automatic recovery of an abandoned run is the explicit exception to download-before-release described below; recovery must stop its owned processes before reuse. -- After taking that lock, every run removes the known `television-source-cli-e2e-*` and `television-web-bundle-*` directories directly under `/tmp`, plus stale per-run scratch. Preserve everything else. This handles residue from test revisions without the helper fixes. -- Keep worker-lifetime helper cleanup, per-run scratch, suitable built-app acceptances, and bounded failure diagnostics. Drop pool drain/recreation and the pending rollout choice. - -Josh then calibrated the lock scope: only our runner and pool scripts touch these workers. Keep their locking correct and simple; do not defend against hypothetical callers or add elaborate mixed-version machinery. The revised proposal uses the existing lock directory and ownership identity, holds it through teardown/download, and makes pool operations respect it. It does not introduce a persistent compatibility fence, separate worker controller, or general API gateway. - -Following round 2, Josh selected automatic recovery and required that a crashed coordinator's still-active run cannot hold a worker forever. The supervisor clarified that his suggested predicate and grace-period mechanism were illustrations: keep a special case if correctness needs it and explain why. Josh also delegated the built-app scope judgment, noting that recurring cleanup reduces the case for a broad conversion. The resulting design and scope below are this contribution's proposal, not a claim that Josh prescribed their mechanics. - -The [proposal](test-harness-resource-cleanup-proposal.md) incorporates the result-download-race note. Its separate file is removed because the correction is now in scope. No spec, proof, or implementation has been changed. - -## Blockers - -| Finding | Current assessment and disposition | -| --- | --- | -| B1: lease redesign is unnecessary for scratch cleanup alone, with unresolved availability and mixed-version consequences | The original scope objection was warranted. Josh now explicitly requires the lock correction. Keep atomic directory acquisition and `lockId`, replace blind reclamation with renewal and cleanup before reuse, audit descendants, and normally release after download. Round 2 below supplies automatic recovery and sizes the availability cost. Keep the fixed results tree rather than a cross-run artifact archive. | -| B2: old branches keep leaking outside the new directory | Address this with Josh's selected recurring pre-run deletion. New locks use `expiresAt: 0`, which the inspected old acquisition code does not reclaim. Older coordinators still release their own locks before downloading; branches take that correction when they integrate main. Round 2 refinement 3 removes the requirement to update all active branches or wait for all old in-flight runs before this contribution's final verification. No permanent old-client exclusion, separate controller architecture, or pool recreation is proposed. Helpers on older branches can still leak; subsequent updated runs reclaim their known residue. | - -## Refinements - -| Finding | Current disposition | -| --- | --- | -| R1: quantify relative benefit | Retain the historical roughly 3.2 GB shared-memory increase and 1.16 GB / 0.079 GB leaked-prefix measurements. PR #32 directly addresses 304-buffer accumulation for registry Vite services. Narrow conversion to two files for their proof boundary and repeated loads; do not claim they are necessary to fix that leak or will save the historical amount of memory. Any incremental resource measurement uses a baseline containing #32. | -| R2: compare pool recreation with legacy deletion | Josh chose recurring exact-prefix cleanup. Remove the alternatives table, recreation recommendation, and unresolved rollout decision. Require whole-worker exclusion and idle confirmation before deletion; the prefix alone is insufficient evidence of disuse. Preserve unrelated `/tmp` contents and persistent caches. | -| R3: commit only to minimum diagnostics | Keep parent start/end RAM, shared memory, temporary free space and OOM-counter observations, plus bounded kernel OOM lines on failure. Retain evidence for recovered retries. Keep the 128 KiB/surface and 8 MiB/attempt limits. Hold one attempt's evidence until retrieval or abandoned-run recovery; disclose discarded orphan reports in a bounded receipt. Detailed probes remain optional. | -| R4: name infrastructure and account for capacity | Retain two private `rose-cleanup` workers and the separate Blaxel coordinator as planned infrastructure, not reserved capacity. Recheck actual pool inventory and provider limits rather than assuming the old 72+3 arithmetic describes today's workspace. All test/build execution remains on Blaxel; private experiments use bounded owned processes and do not invoke isolated-GitHub-only destructive fixtures. | -| R5: disclose remaining unbundled exposure | Retain the named inventory, labeled with its original base revision, and require reconciliation against current main before implementation. Existing deferred cases remain visible; no claim that all of them require Vite or have sufficient headroom. | -| R6: justify helper fixes in addition to scratch | Keep helper cleanup for within-shard worker disposal and callers without a Blaxel wrapper. Preserve once-per-worker/per-version build reuse and clean failed builds. This contribution does not authorize cleanup on the shared development host. | -| R7: sequence PR #9 | PR #9 merged as `f81a2465`. Preserve its theme-reset conversion when integrating current main and reconcile the other theme/auth walks and stable-front proxy against the merged implementation. There is no remaining PR #9 merge-order decision. | - -## Remaining evidence - -Independent re-review should assess the revised scope against Josh's decisions and simplicity calibration: our acquire/release/error paths, result retrieval before release, recovery of uncertain work, and the pool-management script. There is no adversarial API-caller requirement. Live provider behavior and `TMPDIR` placement still need proof-stage experiments. No observed Chromium failure has been attributed to the lock or download races, and no post-#32 experiment establishes the remaining memory benefit of built loading. - -Validation of this revision consists of document, code, policy, and PR inspection plus Git whitespace/diff checks. No tests, builds, browsers, servers, remote experiments, or worker cleanup were run for this revision. The pushed proposal is the next independent-review checkpoint; spec derivation follows convergence. - -## Round 2: automatic recovery and narrower conversion - -Independent round 2 reviewed `d3570bf4`. The [full review](test-harness-resource-cleanup-review-r2.md) is preserved verbatim: **FAIL on undefined recovery of a dead coordinator's lock**. The blocker was warranted. A lock that survives coordinator death without an automatic recovery entry point would steadily remove pool capacity. The proposal now supplies all four missing decisions: - -| Blocker detail | Proposed resolution | -| --- | --- | -| Entry point | The next canonical `acquireAny()` call invokes the common worker lock helper. Pool mutations reuse it. No new recovery service or manual routine for ordinary coordinator crashes. | -| Evidence of coordinator loss | The coordinator renews once a minute through setup, execution, cleanup, and download. A worker-local deadline is the last renewal plus the configured worker timeout plus 15 minutes. Missing renewals beyond that interval mean ownership has been abandoned, whether caused by a crash, suspension, or network partition; no claim of cross-host PID certainty. Expiry permits cleanup, not reuse beneath survivors. | -| Undownloaded reports | Normal runs hold through verified download. After lost renewal reaches the recovery deadline, discard orphan reports during reuse and record their loss with the old identity, process cleanup, and deadline in a bounded recovery receipt. No orphan archive or additional blocking download. A returning coordinator that lost ownership reports infrastructure failure. | -| Capacity cost | At the 480-second cold-worker default, an orphan becomes recoverable 23 minutes after its last renewal; the 240-second default gives 19 minutes. Actual reuse adds the next acquisition attempt and bounded cleanup. A still-running owned supervisor or descendant is stopped by token and audited, so it cannot renew the lock or prevent ordinary recovery forever. | - -One recovery sequence handles quiet and still-active abandoned attempts. The proposal deliberately forgoes early quiet-worker recovery to avoid a second liveness test or report-age clock. It keeps a short OS-released filesystem critical section for competing acquisition/renewal/release, and checks ownership at remote dispatch registration so a delayed API call cannot start obsolete work after recovery. These are races in our own code paths, not defenses against hypothetical callers. - -Two limits remain for correctness: a process whose identity or termination cannot be confirmed cannot be ignored, and a legacy lock without outer-run attribution cannot justify killing by name. Hold and report those repair cases. Updated, attributable runs recover automatically even when their supervisor remains alive. Empty acquisition records with no dispatched work can be reclaimed after a clean worker audit; missing scratch metadata never adds a gate. This distinguishes a real cleanup failure from an ordinary crashed coordinator without multiplying recovery procedures. - -Refinement 1 is resolved by contributor judgment: **five cases in two files**, the settings-persistence and executable-lifecycle cases in `theme-product-acceptance.test.ts` and all three cases sharing `browser-demo-mode.test.ts`'s page helper. The theme product and delivery proofs already describe built application behavior; both files include reload/restart walks and reuse existing helpers. PR #9 owns reset. After PR #32 and routine cleanup, the remaining 15-file migration has diminishing benefit relative to its proof and regression work. Name main's additional isolation/move callers as deferred too. Limit the proposed default to new or substantially changed `ProductServer` browser acceptances, leaving in-process `Server` / `appURLForServer()` paths and Electron seams alone. - -| Refinement | Assessment and local change | -| --- | --- | -| 2: runner scratch needs stronger metadata than legacy prefixes | Accepted. After taking the lock and obtaining a clean worker owner audit, remove every earlier attempt directory in `/tmp/tv-testshards/` except the current one. Missing or partial directory metadata alone cannot block cleanup. Keep the existing path boundary and live-process protections. This does not decide how another run's held lock is recovered. | -| 3: require other branches to update and old runs to finish | Accept removing both pool-wide prerequisites. `expiresAt: 0` protects updated runs from the inspected old acquisition code; only the new helper understands the renewable deadline. Branches integrate the fix normally and final verify uses ordinary acquisition. An old coordinator still releases before downloading, which new code cannot retroactively protect. Respect unexpired legacy TTLs; after expiry a clean worker audit permits reclamation, while unattributable live work holds and reports. | -| 4a: distinguish Chromium 148 investigation from PR #32's 145 evidence | Accepted. State the observed trace version `148.0.7778.96`, the Playwright manifest's expected `145.0.7632.6`, and main's conditional 1223-to-1208 cache aliases. Today's actual worker browser version is unknown. Record the launched browser version and resolved executable/cache path during future Blaxel validation before interpreting memory or temporary-file evidence. No browser alignment change or new experiment is proposed here. | -| 4b: credentials for the Blaxel coordinator sandbox | Accepted. Specify process-environment injection of `BL_WORKSPACE`, an authorized `BL_API_KEY`, and `BLAXEL_TV_GH_TOKEN`, loaded in memory on the launching host. The installed SDK's authentication source and repository token reader support these paths. Do not copy credential files, embed values in commands/scripts/logs, or persist them in sandbox/Git configuration; verify availability before dispatch and end the credential-bearing process after report retrieval. No secret was read, copied, or transmitted while preparing this revision. | - -The revision is ready for independent re-review; the historical FAIL is unchanged until that review converges. No spec, proof, or implementation was edited. No test, build, browser/server launch, remote experiment, or worker operation ran. Live renewal/recovery behavior, browser versions, `TMPDIR` placement, and incremental resource benefit remain unmeasured; the proposal names the private Blaxel validation needed after spec/proof convergence. - -## Round 3: convergence and spec derivation - -The [round 3 report](test-harness-resource-cleanup-review-r3.md) is copied verbatim. It passes the proposal and raises no blocker. Under the workflow's bounded-refinement rule, apply the four substantive improvements without reopening cosmetic review or adding another proposal stage: - -| Refinement | Judgment and disposition | -| --- | --- | -| 1: fixed recovery interval | Accepted. Use 20 minutes, preserving the existing cold-profile allowance, independent of shard timeout. Reuse `--lock-ttl-seconds` for the interval override and derive renewal cadence from it. A healthy coordinator renews throughout ownership, so tying recovery to execution timeout bought no safety. Remove the custom-timeout field/case and the 19/23-minute distinction. This remains a proposed operating allowance to validate on Blaxel. | -| 2: delete a worker that cannot be cleaned | Accepted. Normal test acquisition still requires a clean audit. Explicit pool deletion refuses a valid reservation but may destroy an expired worker despite failed process cleanup. It first removes test-dispatch eligibility so a delayed provider deletion cannot hit a new test run. Failed deletion stays visible to pool management and unavailable to tests. Fresh-only `ensure` does not refill a partly deleted pool. This makes the stated operator repair path usable. | -| 3: live legacy work | Accepted. Skip/report it after TTL expiry and allow its matching-ID exit trap to free it normally. Do not call every live legacy run a repair case or add a second recovery mechanism. An old run that never exits still has the explicit pool deletion path. | -| 4: receipt and provisioning wording | Remove the separate receipt limit; the 8 MiB attempt budget already bounds it. Replace the undefined pool-draining language with delayed publication of test-discovery labels until runtime setup and architecture verification succeed. The existing architecture probe alone is insufficient to exclude a worker in the middle of setup once Node becomes available. Keep incomplete provisioning visible to the manager, with no silent reuse. No new pool-wide coordination protocol. | - -Main at `a8e73fbf` is integrated in `74d22941`, with the policy-level reconciliation in that merge message. Preserve the Vite full-response fix, PR #9's built reset, origin repository selection, single-origin app helper, SDK assets and current proxy API. PR #30 separately fixed demo-mode fixture teardown by closing idle HTTP connections; revise the resource proposal to say so. The five-case conversion is retained for its shipped-asset boundary and repeated loads, with no claim that it cures that flake or produces the historical memory saving. - -### Spec checkpoint - -| Owner | Derived delta | -| --- | --- | -| `specs/arch/testing-policy.md` | Worker-owned reusable builds with Playwright/Vitest teardown and failed-preparation cleanup; built application default limited to new/substantially changed `ProductServer` browser acceptances. | -| `specs/arch/test-runner/blaxel-testshards.md` | Exclusive renewable reservation through teardown and retrieval; fixed recovery interval and existing option; automatic owned-process cleanup and explicit abandoned-report loss; legacy skip; pool provisioning/deletion exclusion; per-attempt scratch and exact pre/post-run cleanup. Replace the TTL-only release promises rather than leaving contradictory authority. Dedicated fresh workers can prove provisioning without recreating the shared pool. | -| `specs/arch/test-runner/reporting.md` | Parent start/end resource observations, failed/recovered attempt retention, missing-evidence honesty, bounded kernel excerpts and sidecars, recovery receipts, and normalized links to retained diagnostics. | -| `specs/arch/test-runner/test-runner.md` | Cross-reference the Blaxel abandonment exception while preserving ordinary live-owner protection; preserve outer attempt attribution through nested scopes. | -| `specs/arch/test-runner/sharded-execution.md` | Carry the outer attempt environment and diagnostic artifact references across shared execution and provider normalization. No planning, shard-count, or retry change. | - -The rewritten spec policy on main governs this derivation. The proposal retains implementation rationale and the future experiment plan; specs own the lasting lifecycle decisions and cross-process/reporting contracts. Mirrored proofs and theme setup descriptions are intentionally unchanged at this checkpoint: they follow independent spec convergence. Product promises, production code, and the five conversion implementations are unchanged by this checkpoint. - -Validation is read-only source/policy review, exact-copy comparison of the independent review, and Git diff/whitespace inspection. No tests, builds, browser/server launches, remote experiments, or worker operations ran. Live recovery, temporary-file placement and bounded evidence transport remain to be proven on committed/pushed Blaxel revisions. The supervisor arranges independent spec review; Josh accepts the spec changes at PR time. - -Primitive surfaced the known recovery decisions and their later correction, alongside unrelated scope matches. Its discovery scan was truncated, so it was not a complete governing-set check. Repository policy and the supervisor's latest instructions governed the assessment; no decision was overridden or reconciled. - -## Spec round 1: convergence and proof derivation - -The supervisor relayed **PASS** on `cd8aa453`. The [complete spec review](test-harness-resource-cleanup-spec-review-r1.md) is copied verbatim. Apply the nonblocking refinements under the bounded rule; no additional spec-review round is requested by this response. - -| Refinement | Assessment and disposition | -| --- | --- | -| 1: terminology | Use lease for coordinator ownership and lock for its directory. Say shard attempt for one shard's execution, retaining attempt for the shard-set attempt. Keep the existing block anchor stable. | -| 2: unreadable records | Use the same clean-audit requirement for acquisition and deletion. Missing ownership evidence authorizes neither signalling by name nor ignoring surviving owners. | -| 3: command-unavailable deletion | Partly accepted. A fresh worker never published for dispatch can be deleted without a lease. Do not adopt the broader inference that a worker absent from coordinator discovery has no live or valid earlier run: the review supplies no provider guarantee for that implication. A previously eligible worker whose ownership check cannot run remains held/reported for operator repair. This preserves the converged no-live-lease deletion rule; it adds no new recovery service or override. | -| 4: helper rule | Trim the spec to worker ownership, publication after success, teardown and the Blaxel backstop. The proof derives pending-build/failure cleanup and real Playwright/Vitest crossings from that rule and ordinary cleanup correctness. | -| 5: operations and output | State idle-pool deletion/refusal consequences, locate resource sidecars and receipts under downloaded artifacts, and name the temporary-cleanup lifecycle step. | -| 6: transitional text | Label the legacy protocol transitional and remove the two non-requirements. | -| 7: testing guidance | Keep actual Node and Chromium descriptor observations. Remove the duplicate placement rule and leave browser-version measurement hygiene in the proposal's experiment plan. | -| 8: sidecar lifetime and wording | Explicitly finalize setup sidecars outside scratch before removal; keep allocation-versus-free-space interpretation in working guidance rather than authority. | - -### Proof checkpoint - -The mirrored proofs now derive the following coverage, with new obligations explicitly marked **test to be written**: - -| Owner | Coverage and cost boundary | -| --- | --- | -| `proofs/arch/test-runner/blaxel-testshards.md` | Contract decisions/order; real guard/registration and filesystem seams; canonical live retrieval/refusal, automatic active/quiet recovery, fresh provisioning and pool deletion on dedicated workers. Planned and targeted paths share small fixtures. Node and actual Chromium descriptor paths establish temporary-directory inheritance. No deliberate OOM, disk exhaustion, shared-pool contamination, or matrix of provider failures. | -| `proofs/arch/test-runner/reporting.md` | Bounded writer and retained-link contracts; real parent/Linux/native-runner capture; downloaded failed and recovered-attempt evidence composed into the same provider runs. Unknown probes, missing observations and lost reports cannot become passing evidence. | -| `proofs/arch/testing-policy.md` | Worker cache/publication contract, real Playwright and Vitest teardown seams with real source-CLI/web output, and failed-launch copy cleanup. The built-app default is honored by test design, not a policy-compliance source scan. | -| Shared-execution and runner proofs | Cross-reference the owner-token, temporary-directory, diagnostic-history and real-plan crossings above. Keep ordinary stale-owner protection and isolated-GitHub fixture restrictions. | -| Theme product/delivery and artifact product proofs | Update the selected five cases' loading boundary and mark adaptation pending. Preserve their user-visible assertions, stable origin/authentication over restart, and the separate Settings development-module walk. PR #9's reset remains unchanged. | - -The main risk worth the live cost is reuse before owned processes stop or evidence reaches the caller. Contracts cover failure permutations, but cannot replace real Blaxel metadata updates, dispatch, renewal, cleanup and download. The two private workers and separate Blaxel coordinator in the proposal remain planned resources, not an assertion that they exist or are available. Actual execution will use committed/pushed revisions and the canonical runner; native retries retain the default budget. Only the existing recovery-interval option is shortened deliberately for crash tests. - -Validation for this checkpoint is document/source inspection, review-copy comparison, Markdown index regeneration, and Git whitespace/diff inspection. No tests, builds, browser/server launches, provider experiments, or worker operations ran. Test and harness implementation has not begun. Independent proof review is the next gate; Josh's spec review remains at PR time. - -Primitive's file check surfaced existing recovery decisions and the correction to the supervisor's illustrative rule, with `truncated: true` from discovery. That was not a complete constraint check; no decision was overridden or reconciled. These proof details derive the existing choices and introduce no new higher-order direction to record. - -## Proof round 1: placement finding and refinements - -The [complete proof review](test-harness-resource-cleanup-proof-review-r1.md) is copied verbatim. Its one blocker is warranted: the proofs ordered automated live qualification and intentionally failing native fixtures without an owning registry/runner placement rule. The worker reads the committed registry; the local replacement-registry hook is not a worker-selection mechanism. - -The proposed correction keeps automated qualification and uses the existing `experiment` kind. Register separate `experiment:blaxel-harness` and `experiment:blaxel-harness-fixtures` surfaces; both stay outside `all` and ordinary CI. The driver runs through the canonical local runner on a dedicated credentialed Blaxel coordinator outside the worker pool and owns a disposable private pool. Its worker selection contains only the native fixtures. This retains repeatability, unlike operator-only live evidence, without a new kind, suite, forwarded registry hook, or general execution-placement framework. Ordinary Linux seams remain in the repository suite. - -The delta is in `test-registry.md` (surface ownership/exclusion), `test-runner.md` (coordinator placement, planned/targeted fixture selection and driver cleanup), and a reference from `blaxel-testshards.md`. It changes no lock, recovery, cleanup or diagnostic promise. It does need independent spec review: the earlier PASS did not cover this missing placement. [tvdev-contribute](/home/user/.agents/skills/tvdev-contribute/SKILL.md) requires “independent agent convergence of every spec delta remains required before proof derivation.” The corresponding repository [workflow](../../specs/spec-workflow.md#Recommended approach) governs that ordering. Josh's human spec review remains at PR time. - -| Finding | Assessment and disposition | -| --- | --- | -| B1: live tests and deliberately failing fixtures have no home | Return the proposed placement above to independent spec review before binding the live proof declarations to it. The existing behavioral live assertions remain pending obligations; no tests or driver implementation are written at this checkpoint. | -| R1: grades and proxy declaration | Accepted. Every affected theme, delivery, artifact and bridge assertion now distinguishes its existing policy-grade walk from the unwritten built-application adaptation. The theme declarations name the real stable-front proxy as harness transport; the coverage model still explains stable origin/authentication across port-0 backend restarts. Reset and already-built walks retain their coverage grades. | -| R2: worker-guard seam placement | Accepted. State ordinary repository test on Linux, with real procfs, signals, OS guard and filesystem. No provider credentials or live-provider acceptance is required. This contribution still executes that test only on Blaxel. The whole-worker audit forfeit remains with live recovery. | -| R3: worker fixture selection | Accepted in the proposed owning specs. A canonical `--tag blaxel-harness-fixture` selection delegates `--suite selection --surfaces experiment:blaxel-harness-fixtures`; `--surface experiment:blaxel-harness-fixtures` selects the targeted path. Bind those commands into the shared live declaration and retrieval assertion after placement convergence. Clarify now that reporting's replacement-registry hook applies only to the local parent/child seam. | -| R4: failed-launch cleanup assertion | Consolidate it into the existing real worker seam, using the missing bundle path and subsequent valid launch within the same fixture; remove the standalone assertion and redundant second contract citation. The review's authority premise is too narrow: the testing policy's timing/cleanup section does say cleanup of long-running modules is core to correctness, below its motion subsection. The consolidation is still worthwhile because the same worker-lifetime crossing can cover the private-copy failure without another fixture or launch sequence. | -| R5: upload path and cleanup timing | Accepted. Add to the existing live retrieval run: uploaded script paths are inside the current shard attempt directory, and retained phases include `temporary-cleanup` with counts and free-space evidence. No new live run. | - -After placement spec convergence, complete the dependent proof binding: name both surfaces and the coordinator host in the shared live declaration, spell out planned/targeted selection in retrieval, and derive selection coverage for exclusion from `all`/ordinary CI and absence of driver recursion. Reporting's live diagnostic assertion shares that same driver. These are pending derivation steps, not silently approved additions to the reviewed proof tree. - -Validation is document/source inspection, exact comparison of the saved review, Markdown index regeneration and Git diff/whitespace inspection. No tests, builds, browser/server launches, provider experiments or worker operations ran. Test and harness implementation remains unstarted. - -Primitive's file check returned existing recovery and placement decisions with `truncated: true`; it was not a complete constraint check. No decision was overridden or reconciled. The latest 20 decisions contained no duplicate of the proposed automated experiment placement; it is recorded as agent-attributed `dec_7893f958`, with the repeatability rationale and operator-only alternative. That record does not substitute for independent spec convergence or Josh's PR review. - -## Proof round 2: convergence and implementation planning - -The supervisor relayed **PASS** on `3fa15d47`; the [complete report](test-harness-resource-cleanup-proof-review-r2.md) is preserved verbatim. The reviewer approved the placement spec delta and explicitly allowed the deferred proof binding as a bounded follow-up without another round. Complete that binding: both experiment surfaces, the dedicated coordinator host, planned/tag and targeted/surface commands, and one source-level registry/driver selection contract. Reporting's live assertion shares the same driver and placement. No new live scenario, suite, kind or test hook is introduced. - -| Refinement | Assessment and disposition | -| --- | --- | -| 1: general selector description | Accepted. State package/runner/tag planned selection alongside the existing target path; this documents the current routing used by the reviewed placement. | -| 2: broad-run guardrail description | Accepted. Cross-reference the Blaxel harness surfaces and their exclusion from `all` alongside the other opt-in suites. | -| 3: minimum fixture shape | Accepted in the shared proof declaration and implementation plan. At least two native files are needed for the reviewed two-shard, file-atomic fixture run. This is derived test design, so it does not need a permanent minimum inventory rule in the registry spec. | - -The work is not clearly one slice. The [implementation plan](test-harness-resource-cleanup-plan.md) proposes three coherent slices: worker-owned builds plus the five built-app conversions; bounded diagnostic capture/reporting; then exclusive leases, recovery, scratch/pool operations and the shared live qualification. Keeping the entire remote ownership lifecycle together avoids partially integrated exclusion rules. Separating diagnostics and helpers makes their real framework/process boundaries reviewable without repeating the paid-service scenarios. Slice 3 explicitly owns the pending remote diagnostic evidence and killed-worker backstop from the earlier slices. Each slice has its red/green criteria and expected baseline; the integrated result has a separate review and full gate. - -Every test/build runs on a pushed revision on Blaxel. All worker-affecting validation uses `rose-cleanup`; the driver runs on `tv-cleanup-coordinator-rose` outside the pool. The plan also chooses the same private two-worker pool for complete `all` verification using the existing shard-count override, replacing the proposal's earlier shared-pool final-run arrangement. The complete selection still qualifies under the runner spec; private capacity and timeout headroom are unmeasured and must be reported if they block the gate. No silent shared-pool fallback is authorized by this plan. - -Validation for this planning checkpoint is document/source inspection, exact review-copy comparison, Markdown index regeneration, and Git diff/whitespace inspection. No tests, builds, browser/server launches, provider experiments or worker operations ran. Implementation waits for the supervisor-arranged independent plan review. Josh's spec review remains at PR time. - -Primitive surfaced the already-recorded opt-in experiment choice after the prior turn. That was consistent context but added no new requirement. Its edit hook reported a truncated governing-set scan, so it did not establish a complete constraint check; no decision was overridden. This turn follows the settled direction; the slice ordering and private validation arrangement are contribution-scoped execution choices, so no new durable decision was recorded. - -## Plan round 1: full-gate placement and refinements - -The [complete review](test-harness-resource-cleanup-plan-review-r1.md) is preserved verbatim. Its one blocker is warranted: the two-worker full gate had no credible setup/retry allowance, and deferring that known problem until the end of implementation would only delay completion. The three slices, proof assignments and intermediate baselines remain unchanged. - -Read-only aggregation of `test/timing-baseline.json` confirms 311 Blaxel file medians totaling 882.506 seconds, or 441.253 seconds per worker when divided over two; `e2e:browser-app` contributes 376.485 seconds. These are historical measurements through 2026-09-29, not a measured duration or guaranteed timeout for the eventual tree. The coordinator's cold profile gives each worker process 480 seconds, including setup, and `scripts/test/cli.mjs` forwards no worker-timeout override. That is sufficient evidence to reject the two-worker gate budget without executing it. The documented tier has no allowance for a standard-size private pool beside the shared pool; actual occupancy still needs checking before provisioning. - -Restore ordinary full verification to `npm run verify -- blaxel --pool poc`, at the standard 36 shards, invoked from `tv-cleanup-coordinator-rose` after private qualification and deletion of both private workers. The previous all-private gate was my overly broad reading of the worker-experiment restriction, not a requirement from Josh. All tests, lint, type checks and builds still run on Blaxel at pushed revisions; every fault, interruption and cleanup experiment remains private. The shared pool sees only ordinary full verification through the implemented runner. This restores the converged proposal's validation arrangement without changing a spec, proof obligation, timeout, retry budget or shard default. - -| Refinement | Assessment and disposition | -| --- | --- | -| 1: large surface expansion | Accepted. Keep focused cases, complete affected files and directly affected helper callers on the private pool. Leave whole large surfaces, including `e2e:browser-app`, to the standard-shard integrated gate. State that `--surface` takes the one-worker targeted path even with `--shards`; do not imply the private pair splits it. | -| 2: name option forwarding | Accepted. Add `scripts/test/cli.mjs` and explicit `--lock-ttl-seconds` forwarding to slice 3's implementation area, with CLI delegation coverage and the existing live recovery run. Omitting the override retains the specified default. No new worker-timeout control. | -| 3: request inventory mechanism | Accepted. Use Playwright request events within the existing five walks. No extra launch or inventory harness. | -| 4: capacity arithmetic | Accepted as a planning bound, not a current inventory. The documented 72 shared workers plus two private workers and one coordinator reach 75 exactly. Check current workspace occupancy/limits before provisioning; report insufficient room without altering shared workers. | - -Validation for this revision is whole-review reading and exact-copy comparison, source/spec inspection, arithmetic over the committed timing JSON, and Git diff/whitespace checks. No test command, build, browser/server launch, provider call or worker operation ran. Implementation remains unstarted and waits for the supervisor-arranged plan re-review. Coordinator capabilities and current capacity remain unmeasured; Josh's spec acceptance remains due at PR time. - -Primitive's edit hook reported a truncated governing-set scan, so it did not establish a complete constraint check. No decision was overridden or reconciled. This revision follows the converged behavior and corrects contribution-scoped execution choices; it establishes no new durable team decision. - -## Plan round 2: convergence and slice 1 - -The [complete PASS report](test-harness-resource-cleanup-plan-review-r2.md) is preserved unchanged. Accept its two bounded refinements: slice 3 adds an explicit supplied/default CLI-argument contract for `--lock-ttl-seconds`, and capacity planning no longer equates the documented concurrent-shard limit with a count of idle sandboxes. No slice boundary or proof obligation changes. - -Read-only provider inventory found 143 deployed `poc` sandboxes and none of the named private hosts. The documented 72-worker inventory is therefore not current. Blaxel accepted creation of `tv-cleanup-coordinator-rose`; the canonical pool manager, running there at pushed `04a6b275`, provisioned `tv-testshard-x64-rose-cleanup-01` and `-02`. Each uses the named Playwright image, 8192 MB and Node 24.22.1/npm 11.19.1, with x64/Linux verified on the workers. API-key authentication and the GitHub credential are available; values were passed only through the SDK process environment. The provider accepted this capacity, but its account-specific quota and idle-sandbox accounting remain unknown. No shared worker was modified. - -The first red checkpoint adds a real Playwright/Vitest worker fixture and parent seam. Each worker uses real source CLI and versioned web output in two cases, observes reuse after per-case cleanup, and leaves its outputs for the parent to inspect after normal teardown. An absent bundle exercises failed-launch private-copy cleanup; a later worker must receive different paths. Nested fixture retries are zero to observe exactly two cases; the canonical parent retains its default retry budget. Test execution has not yet occurred at this checkpoint. All subsequent commands and evidence are recorded below. - -## Slice 1: implemented for independent review - -The [slice 1 evidence record](test-harness-resource-cleanup-slice-1.md) contains the red/green revisions, exact selections, canonical run directories, request inventories, failed intermediate attempts and remaining limits. The code checkpoint is `38976455`; the following proof/document update changes no executable behavior. - -The shared build owner now publishes only successful preparation, reuses each worker's source CLI and each versioned web build, removes failed output and waits for preparation during teardown. Real Playwright and Vitest worker fixtures close that lifetime after server cleanup. Per-launch cleanup is registered before preparation and removes failed private copies and homes. Existing direct and indirect builder callers receive the appropriate fixture registration. The selected two theme and three demo-mode cases now load built assets while retaining their product assertions; PR #9's reset and the separate Settings development walk remain unchanged. The theme restart path uses the existing stable-front proxy, and the helper supplies the built unsupported-URL view at the server's static path. - -Four contract cases and the real two-framework seam have intended red/green evidence. The five converted walks first failed on their observed development-module requests, then passed on built application requests. Both affected browser files and ten focused caller selections passed, as did remote lint/type checking and the final cache/startup-disposal checks. Completed green runs needed no recovered retries and reported no process leaks. The final proxy check had one separate incomplete attempt when the coordinator's Git fetch stalled before acquisition; the confirmed fetch child was stopped, its failure retained, and the unchanged selection passed on repetition. The evidence record distinguishes that infrastructure interruption from test failures. - -All execution occurred on the named private Blaxel hosts at pushed revisions; nothing ran on the shared development host and no shared worker was modified. The five mirrored proof files now cite the implemented, passing coverage. The optional Firefox project was unavailable; no new Firefox validation is claimed. Slice 1's killed-worker backstop, diagnostic transport, renewable worker ownership and recovery remain assigned to slices 2 and 3. Full verification and integrated review are still pending. The private hosts remain allocated for those slices. - -The supervisor arranges independent review of this slice before further implementation. No new durable team decision was made; this work implements the converged proposal, specs, proofs and plan. - -## Slice 1 round 1: convergence and slice 2 - -The supervisor relayed **PASS** at `036d0491`; the [complete review](test-harness-resource-cleanup-slice-1-review-r1.md) is preserved unchanged. Apply the proof-only refinement by attributing per-case server disposal to the real seam, and clarify that the final static log ends with the separately recorded fetch interruption. Keep the second framework invocation: it directly exercises the reviewed later-worker lifetime assertion, and removing it now would change that proof's scope. This optional cost reduction does not block convergence. - -The reported 143 deployed shared sandboxes remain an operator question for Josh and an integrated-review documentation item. Do not infer quota semantics or change the pool. Slice 2 implements bounded parent diagnostics, retained links and retry history; live provider recovery/diagnostic qualification and the lease/download race remain explicitly pending until slice 3. - -## Slice 2: implemented for independent review - -The [slice 2 evidence record](test-harness-resource-cleanup-slice-2.md) records red/green commits, exact canonical selections, all intermediate failures and the remaining boundaries. The final code checkpoint is `0e82df7f`; the following proof/document update changes no executable behavior. - -The shared collector retains a baseline before preparation and final observations after cleanup for planned standalone, combined Vitest and targeted execution. The targeted Vitest path uses the existing attempt recorder so recovered failures trigger kernel capture. Bounded serialization and download preserve identities, unavailable reasons, counter resets, exact execution intervals and native evidence, with no sample payload in compact summaries or timing events. Aggregation keeps earlier attempt links without replacing the final native outcome. The real-reader seam caught an installed util-linux parser incompatibility; the implementation now reads an enclosing whole-second UTC window and filters its output to the exact observation interval. - -Five collector/interval contracts, two retained-link contracts, four parent cases and the combined-project seam have passed. Complete reporting, provider and shard-worker files passed, followed by focused checks for the kernel-reader corrections. Lint and type checking passed on the private coordinator at the final code checkpoint. Every test used the canonical runner and private pool at a pushed revision; none ran on the shared development host or shared pool. All executed runs reported zero process leaks and infrastructure failures. Temporary diagnostic probes used during kernel-reader debugging are removed. - -The reporting proof now cites the implemented bounds, parent and link coverage. Its live download/reuse and recovery-receipt assertion stays pending for slice 3. The current lease/download race, worker recovery, scratch reclamation, persistent-worker retention, full verification and integrated review are still outstanding. The private hosts remain allocated. The supervisor arranges independent review before slice 3 begins. No new durable team decision was made. -# Slice 2 convergence and slice 3 start - -The supervisor relayed PASS at `296391c0`; the complete review is preserved in -`test-harness-resource-cleanup-slice-2-review-r1.md`. Accept the bounded -refinements: reuse the timing-provider constant, resolve targeted Vitest configs -from the declared surface cwd, and reconcile recovery receipts with slice 3's -actual records. Passing surfaces intentionally retain compact start/end sidecars -within the existing per-attempt budget; this aids comparisons without a sampler -or cross-run archive. - -Slice 3 starts with real generated lock-command checks on disposable Linux roots. -The first checkpoint only extracts the existing command without changing its -timestamp-only behavior; the new assertions require renewable ownership and -confirmed quiet-orphan recovery. Execution is pending on the private Blaxel pool. - -## Slice 3 first red and Primitive observation - -At `1320c957`, private canonical run -`.test-runs/2026-10-27T18-41-03-525Z-p19298-r7dc71e154247c385` -reached both native lock assertions and failed them with completed infrastructure: -the timestamp-only record lacks the renewable identity and cannot recover an -expired renewable lease. Defaults were unchanged: no unit retries, two allowed -infrastructure retries (none used). - -Primitive flagged `dec_d81a353e` during pool-lifecycle edits. The ordinary CLI was -absent; npm returned ETARGET and its installed launcher failed with a dynamic -require error. A temporary loader recovered the decision text: it repeats the -earlier lock directive, including the literal no-expiry language, without the -later reviewed automatic-recovery clarification. Recorded the check as followed: -normal runs retain ownership through download; expired renewal authorizes owned -process cleanup, never reuse beneath live work. The converged specs govern that -clarification. The supervisor explicitly directed no further Primitive debugging. -Its partial/truncated decision scan is not a complete authority check. No tests, -builds or application processes ran on the shared host. - -The next private run (`2c0f5d6b`, -`.test-runs/2026-10-27T18-50-52-346Z-p19842-r74788ac655aa50a6`) did not qualify: -Vitest transformed dynamic imports inside the serialized command, and post-run -cleanup mistook overlay backing-device IDs for mount boundaries. Both workers -retained their leases; no shared workers were used. Direct read-only inspection -showed device 19 for `/tmp` directories and device 20 for regular files, with one -overlay mount in `/proc/self/mountinfo`. Correct the serialization and inspect -actual mount records while retaining directory-boundary rejection. Do not alter -the held deadlines; subsequent work will use ordinary expiry recovery. - -A further Primitive warning named `dec_8e3d38a8`, check -`td7crdek1b1n4dhajyww5p174s8h23bq`. Its decision text was not read. Per the -supervisor's direction, record the unread warning and continue with the converged -ordinary Linux fixture seam and private live-driver proof; do not debug the CLI. - -## Slice 3: implemented for independent review - -The final executable checkpoint is `45202ee9`. The [slice 3 record](test-harness-resource-cleanup-slice-3.md) preserves the original renewable-lock red, intermediate serialization/overlay/checkout-reset failures, focused green runs, live-driver corrections, and the late prepared-but-undispatched evidence regression and fix. The complete slice 2 review remains preserved and all three bounded refinements are addressed. - -Ownership now serializes through a short worker guard, renews through downloads, registers dispatch before checkout, audits and stops attributed descendants, and releases only after retained evidence. Expired renewal performs the same cleanup for active and quiet abandoned runs and keeps a bounded receipt. Known temporary prefixes and attempt scratch are reclaimed under that ownership; pool publication and deletion obey the same exclusion. Older timestamp-only records retain the reviewed transition behavior. The preparation-retention fix distinguishes requesting preparation from requesting execution and retains available artifacts even when a required log read fails. - -Seven real Linux lease seam cases and ten lifecycle/cleanup/pool contracts have passed, alongside affected provider, discovery, stale-owner, registry, diagnostic, shard-worker and process-helper files. Lint and type checking passed at the final executable checkpoint. The final full live driver passed on its first native attempt in 4m 31s, with zero outer failures, recovered retries, process leaks or infrastructure failures. Its intentional failed/recovered native fixtures, competing and absent-pool refusals, killed-coordinator reports, receipt links, and final cleanup are retained. The proof citations now name the implemented coverage, resolving the temporary live-provider obligations from slices 1 and 2. - -Both private workers were deleted and confirmed absent; the dedicated coordinator remains for the integrated gate. No tests or static checks ran on the shared host, and no shared-pool experiments occurred. Independent slice review and then integrated review/full verification remain; the draft PR is not ready to merge. No new lasting team decision was made. Primitive's unread conflict ID and incomplete hook scans are recorded in the validation record, and its CLI received no further debugging. - -## Slice 3 round 1: reusable qualification - -The [complete review](test-harness-resource-cleanup-slice-3-review-r1.md) is preserved -unchanged. Its one blocker is warranted: a committed acceptance must remain -runnable after this contribution's personal sandboxes are deleted. The placement -spec already requires a dedicated coordinator generally; this corrects its -implementation without changing the approved placement or lifecycle. - -| Finding | Assessment and disposition | -| --- | --- | -| B1: fixed personal coordinator, pool and fixture names | Require non-empty `TV_BLAXEL_HARNESS_COORDINATOR` and `TV_BLAXEL_HARNESS_POOL` in the CLI and driver. Preserve local-only execution, the shard-token refusal and the real coordinator-label check. Derive leftover and sentinel names from the configured pool. Remove every personal literal from the driver and fixture input. Document invocation and private-pool deletion in the harness README. | -| R1: stop renewal after one transport failure | Retain the reviewed fail-closed behavior for this correction. A failed exec currently supplies no reliable ownership evidence, whether from transport, guard timeout or worker-command failure. Retrying safely would require distinguishing those cases and bounding uncertainty against the last confirmed deadline, plus new failure coverage. That may improve availability, but the unexplained 60-second lapse does not establish a transport cause. The cost remains explicit: one failed renewal can fail the shard and hold the worker until ordinary recovery. No claim of transport-error tolerance is made. | -| R2: profile-independent recovery default | Accepted. Extend the existing forwarding contract to pin 1200 seconds for `normal`, `cold` and `repair`. No default changes. | -| R3: remove checkout's results reset | Defer this optional simplification. It touches repository switching and its seam; that spec also requires live switching between two repositories. The existing preservation path is covered and passed. Expanding this name-configuration correction would buy no additional lifecycle guarantee. | - -The mode filename is neutral and fixed per worker rather than pool-derived: -`/tmp/tv-blaxel-harness-mode.json`. Every worker is freshly created for one private -pool and deleted afterward, so the input cannot collide across pools. This removes -the personal dependency without adding production environment forwarding solely -to locate fixture data. Pool-derived cleanup fixtures and sentinels remain distinct. - -The new placement acceptance spawns the real CLI with authored environment values -and the existing guarded dry-run hook. It proves configuration and refusal before -preflight, not provider identity. The complete live driver remains responsible for -the real API, labels and configured pool. The default assertion is a source check, -with actual lease behavior still covered by the existing Linux and live evidence. - -Validation is recorded in the [slice 3 record](test-harness-resource-cleanup-slice-3.md). -Only the named private hosts are used. Integrated reconciliation, full verification, -docs preparation and ready-PR status still follow slice convergence. - -At `7b0213bb`, the focused placement red became green, both affected whole files -passed, and lint and type checking passed on Blaxel. The complete live qualification -then passed on its first attempt in 4m 56s with fresh private workers. Both workers -were deleted and confirmed absent; the complete reports are retained. No shared -host tests or shared-pool experiments ran. The correction is ready for re-review; -the subsequent documentation/proof commit changes no executable behavior. - - -## Slice 3 round 2: convergence and integrated validation - -The [complete review](test-harness-resource-cleanup-slice-3-review-r2.md) passes -`dc1d9f2e` with no blocker. Both observations are accepted under the bounded rule: -add one comment explaining why the two immediate CLI dry runs deliberately clear -the inherited shard token, and count shards that report `Lease renewal lost` in -the ordinary 36-shard gate. No lifecycle, default or retry policy changes follow. - -The [integrated record](test-harness-resource-cleanup-integrated.md) carries the -current-target reconciliation and full-gate evidence. The supervisor will arrange -integrated review after the gate result. Docs preparation and marking PR #12 ready -remain later steps. - - -## Integrated checkpoint: full gate passed - -The [integrated record](test-harness-resource-cleanup-integrated.md) reports the -whole-tree reconciliation against current main `a8e73fbf` and the standard -36-shard gate at pushed `7c66d160`. It passed: 22 surfaces, 3,705 tests passed, -eight skipped, zero failed or recovered tests, zero process leaks or infrastructure -failures, and zero renewal-loss shard attempts. Static verification also passed. -All results are retained locally; private infrastructure cleanup is recorded. - -Automatic attestation publication failed non-fatally with GitHub HTTP 403. Four -worker candidates without the configured nvm runtime were skipped and replaced; -they were not repaired. Neither issue is hidden by the passing gate. The exact -tested commit is `7c66d160`; this following working-record update runs no tests. -The contribution now stops for the supervisor's independent integrated review. -Docs preparation and marking PR #12 ready have not begun. diff --git a/docs/working/test-harness-resource-cleanup-slice-1-review-r1.md b/docs/working/test-harness-resource-cleanup-slice-1-review-r1.md deleted file mode 100644 index f3daa46e..00000000 --- a/docs/working/test-harness-resource-cleanup-slice-1-review-r1.md +++ /dev/null @@ -1,57 +0,0 @@ -# Slice 1 implementation review, round 1: test-harness resource cleanup (PR #12, 036d0491) - -**Verdict: PASS.** Slice 1 delivers what the plan assigned it: reusable source-CLI and versioned web builds now belong to the runner worker that created them, are published only after preparation succeeds, survive per-case cleanup and are removed by real Playwright and Vitest worker teardown; failed preparation and a failed per-launch copy leave nothing behind; and the five selected walks load the built application through the product server, with the two theme walks keeping their origin across port-0 restarts through the stable-front proxy. The tests prove what the proofs order, at the declared shapes, with no mock on the paths that matter. The validation record is accurate: every row of its run table matches the run directory it names, by commit, counts and outcome, and its two non-green events (a setup failure and a stalled fetch) are described as exactly what they were. No production file changed. Three refinements follow, none blocking, plus one observation outside the slice that the supervisor should see. - -- Reviewed: `git diff 04a6b275 036d0491` in full (helpers, fixtures, the two new tests, the five converted walks, eleven registration edits, the proof regrades, the record); the plan's slice 1; `^worker-build-lifetime`, `^product-acceptance-built-app` and the testing policy's mocking, shape and cost rules; the proof assertions the tests cite; the server's static and view routing; the runner's registry; and the retained run directories and static logs the record names, which exist in this worktree. -- Branch state: contains `origin/main` (`a8e73fbf`); the copied round 2 plan review is identical to mine; the plan edits since `04a6b275` are the two round 2 refinements and a status update. -- I ran nothing. I read run summaries, native reports and logs with small scripts; nothing was built, tested or changed. - -## What the slice proves, against the proofs - -**`^testing-t-build-lifetime` (contract, `test/repo/worker-builds.test.ts`).** Four cases over a real temporary filesystem with recording builders: a concurrent request for the same key gets the same pending promise and the builder runs once; distinct keys get distinct directories; a failed preparation removes its partial output, is not published and permits a fresh build; teardown waits for pending preparation, fulfilled or rejected, then leaves the root empty. These match the contract's text, with one clause that belongs elsewhere (refinement 1). Red evidence: four failures at `2460d06a` on retained output; green at `8bfd952f`. - -**`^testing-t-build-worker-seam` (seam, `test/node/worker-build-lifetime.e2e.test.ts`).** Each of Playwright and Vitest is spawned for real on an authored two-case fixture under a private `TMPDIR`, with configs generated into that directory so the registry sees no unregistered surface (the first attempt's validation failure at `564a8edb` is correctly recorded as setup, not red). The fixture starts a real product server from the source-CLI build and fetches its document and a script; builds a real versioned web bundle and reads its entry; asserts one shared source build after case cleanup, the bundle still present and the case's home gone; and in its first case makes the per-launch copy fail on an absent bundle and asserts no `television-product-e2e-*` remains before a valid launch. The parent asserts exit 0, two observations identical but for case number, no owned-process survivor, and no `television-source-cli-e2e-*`, `television-web-bundle-*` or `television-product-e2e-*` left in the private root after teardown. That is the crossing the proof orders: real runner, real builders, real server, real framework teardown, nothing mocked. Red at `74485d22` and `2460d06a` (four retained directories), green at `8bfd952f`. - -**The implementation behind them.** `WorkerBuilds` (`test/helpers/worker-builds.ts:18-44`) creates and registers the directory before the builder runs, publishes the directory only when the builder's promise resolves, deletes the key and the directory on rejection, refuses new requests once disposal has begun, and waits for every pending build before removing. `getSourceCLIEntry()` now returns its entry only after the build and the runtime-asset copy succeed (`test/helpers/product-server.ts:69-77`), which is the original leak closed at its cause. The web builder keeps its version keys and serialized environment override. Per-launch cleanup is registered before configuration or preparation can fail; disposal during preparation waits for it rather than racing the home's removal; a failed bundle copy removes its own run directory (`:85-105`, `:221-230`). Playwright's shared registration gains an automatic worker fixture that disposes servers first and removes builds only if that succeeded, so a server that could not be stopped keeps its files (`test/helpers/playwright.ts:8-17`); Vitest callers get the same through `test/helpers/vitest.ts`. Every caller of `launchProductServer` or `buildVersionedWebBundle` in the tree imports one of the two registrations (38 callers; the only exception is the fixture body, which the fixture files cover). - -**The five walks (`^product-acceptance-built-app`; the regraded theme, delivery, artifact and bridge citations).** Demo mode navigates to the server URL and asserts the built boundary on every load; the theme walks navigate to the stable-front proxy, retarget it after `restart()`, and assert the boundary after the first shell. `recordApplicationRequests` records main-frame request paths without query values or headers, asserts no `/packages/`, `/@vite/`, `/@id/`, `/@fs/` or `/node_modules/` path, a root document and a built `/assets/*.js` script, and attaches the inventory on exit, pass or fail. I decoded the attachments from the green full-file theme run: two inventories, 27/26 and 38 records, three root documents, zero development paths, as the record states. The unsupported-URL view is now copied under the runtime's static web root, which is where production serves it: the server mounts only `artifact-missing` and `markdown` explicitly and serves everything else from `staticDir`, into which the CLI build copies the web dist with its `views/url-unsupported` (`packages/server/src/server.ts:320-339`). Red at `d43623ba` on all five cases (development modules requested), the persistence 404 at `a2fd5a1e` fixed by that placement, green at `3156e303`; the reset walk and `settings.test.ts` are untouched. - -**Regrades.** Each changed citation now names the test that passed, and the coverage models drop their "to be written" notes. The full theme file's one skip is the Firefox Markdown case, which the web Playwright config skips when the Firefox executable is absent, and the record says so. - -## The record, checked - -| Claim | Evidence | -| --- | --- | -| 25 runs, each at the named commit with the named result | All 25 summaries present locally; `git.commit` matches each row; counts match (`testsFailed` 1/4/1/2/3/1 on the red rows; zero failures, zero recovered retries, zero leaks, zero infra failures on every green row). | -| The incomplete proxy run was a stalled coordinator fetch, not a test result | Its summary: `surfacesRun 0`, `infraFailures 1`; the static log shows `git-remote-https died of signal 15` after the author stopped the child. The repeat passed. | -| Lint and type-check passed at `38976455`; `502ba255` had one type error | `.test-runs/slice-1-static/502ba255.log` shows the `intervals` error; `38976455.log` shows both commands clean before the two final Blaxel runs. | -| Default retry budgets kept | The coordinator lines show `--target-retries 0` for `unit:root` and `--test-retries 2` for `e2e:node`, with `--retry-infra 2`. | -| Nothing on the shared host; private pool only | Every coordinator line names `--pool rose-cleanup`; every path is under `/workspace/television` on the coordinator. | - -## Refinements (none blocking) - -1. **One contract clause is proven by the seam, not the contract.** `^testing-t-build-lifetime` says "Per-case server disposal cannot invalidate the worker's reusable build", but `test/repo/worker-builds.test.ts` has no server; the fixture body of the seam proves it (shared outputs present after `server.dispose()`). Move the sentence to `^testing-t-build-worker-seam`, or cite the seam for it. Proof-only. -2. **The seam runs each framework twice; once would prove the same.** The second nested run exists to show "a later worker owns a separate lifetime", but its separate paths follow from its private `TMPDIR`, and the lifetime claim that matters, nothing left after teardown, is proven by the first run. Dropping the loop halves four source-CLI builds and four Vite builds per execution of this test. Optional; the plan accepted a real build and the surface is remote-only. -3. **One line in the record for the last static log.** `38976455.log` ends with the stopped fetch's stack trace, which a reader opening it cold could take for a static-check failure. The record already explains the event in prose; a clause pointing at it next to "Static logs are retained" would save the reader the inference. - -## Outside the slice, for the supervisor - -The response records a read-only provider inventory of **143 deployed `poc` sandboxes** against the 72 the Blaxel spec documents (`blaxel-testshards.md:25`). Slice 1 is unaffected, but it bears on two things already in play: the plan's tier arithmetic (idle sandboxes evidently do not count against the 75-concurrent-shard limit, or the limit is not what the spec says) and the spec's pool-size statement, which is stale either way and should be reconciled at the integrated review. Whether 71 extra sandboxes are intended capacity or accumulated residue is a question for Josh or whoever operates the pool, not for this contribution. - -## Reviewer checklist - -- **Proof coverage.** Both slice obligations discharged at their declared shapes with the cited tests; the eleven regraded walk citations name passing tests; coverage models updated. -- **Mocking and boundaries.** The contract's builders are the declared substitutes; the seam and the walks have no mock. Clearing `VITEST` for spawned servers is not a mock: it removes the guard that would stop the CLI's entry under Vitest (`packages/cli/src/index.ts:2283`) and is used by ten existing tests on `main`. -- **Honesty of evidence.** The record distinguishes setup failures, red evidence, stalled infrastructure and skips, and claims no Firefox execution. -- **Scope.** Test helpers, fixtures, tests, proofs and working documents only; no production, dependency, timeout or retry change. -- **Baseline.** Green for its obligations; the killed-worker backstop remains assigned to slice 3 as the plan states. - -## Evidence and limits - -- Read every changed file and the full current `worker-builds.ts`, `product-server.ts` launch path, both registrations, the request helper, the fixtures and both tests; cross-checked all 25 run summaries, two native reports and two static logs; decoded the request-inventory attachments. -- Not established: anything about the private workers beyond what the logs show; the exact Vitest behavior of a worker-scoped fixture across files under `pool: forks` (the seam's green run on both runners is the evidence that teardown fires where it must). - -## Housekeeping - -- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel beyond scripts that read JSON and logs already on disk. `git fetch origin` updated remote-tracking refs. -- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-slice-1.md b/docs/working/test-harness-resource-cleanup-slice-1.md deleted file mode 100644 index 4e1067ab..00000000 --- a/docs/working/test-harness-resource-cleanup-slice-1.md +++ /dev/null @@ -1,66 +0,0 @@ -# Test-harness cleanup: slice 1 evidence - -Owner: rose. Draft PR #12, branch `thopter/test-harness-resource-cleanup`. Derived from the converged [plan](test-harness-resource-cleanup-plan.md). This record covers worker-owned builds and the five built-application walks. It does not claim the diagnostics or remote worker lifecycle from slices 2 and 3. - -## Changes and review scope - -`WorkerBuilds` owns temporary directories before preparation starts, shares pending and successful results by key, removes failed output and permits retry, and waits for preparation before worker teardown removes the remaining output. Both real builders use it. The web builder retains serialized environment overrides and version-keyed reuse. Source CLI preparation now returns its entry only after the build and runtime assets succeed. - -Playwright's existing shared registration now has a worker fixture in addition to its per-case server cleanup. Vitest callers use a small worker-scoped fixture registration. Normal server disposal preserves reusable builds. Worker cleanup stops registered servers before removing builds; failed server cleanup remains registered and prevents build removal. Per-launch cleanup is registered before configuration or preparation, and a failed private bundle copy removes both that runtime and its launch-owned home. - -The five selected walks load built assets: persistence and executable lifecycle in `theme-product-acceptance.test.ts`, plus all three demo-mode cases. The theme walks keep their browser origins through the existing stable-front proxy while each replacement backend binds port `0`. Existing product, appearance, motion and authentication behavior is preserved. The unsupported-URL document is copied under the test runtime's `web/views/url-unsupported` static path, matching the production package layout. PR #9's reset test and the separate Settings development-module walk are unchanged. - -Five already-built browser update/gate files and two Node update/reload files also change their test registration so their versioned web outputs receive teardown. Their loading and assertions do not change. The other Vitest registrations are the source-CLI proxy and ACP callers, foundation-copy tests, and build-config integrity. No production application file, dependency, timeout default or retry default changes. - -## Placement and red/green - -Every test ran from `/workspace/television` on `tv-cleanup-coordinator-rose`, using the canonical Blaxel runner and private pool `rose-cleanup`. Native execution ran on `tv-testshard-x64-rose-cleanup-01` or `-02`, at committed, pushed revisions. No tests, builds, lint, type checks, browsers or servers ran on the shared development host. The shared pool was not used or modified. Default canonical retries were retained: two for e2e and zero for unit surfaces, plus two infrastructure retries. The nested worker fixtures deliberately use zero retries so each child records exactly two cases; their canonical parent retains the e2e budget. - -- `564a8edb` stopped at registry validation before execution because two static nested configs looked like unregistered surfaces. This is setup failure, not red evidence. `74485d22` generates the native configs inside the acceptance's private temporary directory, following the existing nested-runner arrangement. -- At `74485d22`, the Playwright child successfully built and used the real output in both cases, but the parent found four retained harness directories after worker exit. The Vitest case established the same failure at `2460d06a`. All three canonical attempts failed on that assertion, with no process-leak or infrastructure failure. -- The four cache contracts at `2460d06a` failed on retained partial/completed output. Their builders are the proof-declared substitutes and use a real temporary filesystem. The lifecycle seam, rather than these callbacks, establishes usable real CLI/web output and actual framework teardown. -- At `8bfd952f`, all four contracts, both framework seam cases, and the existing startup-disposal/proxy file passed. Each framework ran two worker lifetimes; cases reused output within a worker, later workers had separate paths, failed-copy output disappeared before the valid launch, and teardown left no owned build or launch directories. -- At `d43623ba`, all five selected browser cases reached and failed the new request-boundary assertion: development modules were still requested. At `a2fd5a1e`, the executable walk passed but persistence failed on a 404 for the unsupported-URL view. The helper had copied it to the sibling-view location even though the server serves this document through the web static root. `3156e303` corrects only that destination. The narrow persistence run, whole demo file and whole theme file then passed. -- Inspection of the repository TypeScript target prompted `502ba255`: explicit constructor field assignments replace parameter properties, and test rendezvous promises use ES2022-compatible construction. This changes no lifetime behavior. Remote lint passed at that revision; type checking found that the adapted startup-disposal assertion had used Playwright’s plural `intervals` instead of Vitest’s singular `interval`. `38976455` corrects the option. Both `npm run lint` and `npm run type-check` then passed on the private coordinator at that pushed revision, and the cache-contract and startup-disposal files passed there through the canonical Blaxel runner. Static logs are retained locally under `.test-runs/slice-1-static/`; `38976455.log` ends with the later stopped-fetch stack trace described below, after lint and type checking had passed. - -The run table below records the native evidence and exact canonical selections. Each command has the prefix `npm test -- blaxel --pool rose-cleanup`; the `--run-dir-output` receipt used for orchestration changes no selection. Every run directory is under `/workspace/television/.test-runs/` on the coordinator and has also been copied under `.test-runs/` in the contribution worktree. `summary.json`, `results.json`, `events.ndjson`, the provider request/report/setup/shard files and native JSON are retained for executed runs. The interrupted fetch has only canonical summaries/events and its coordinator log; it never produced a provider or native report. The native JSON embeds request inventories, so they remain available after later worker checkout or cleanup. - -| Commit | Canonical selection after the command prefix | Result | Run-directory name | -| --- | --- | --- | --- | -| `74485d22` | `--file test/node/worker-build-lifetime.e2e.test.ts --grep 'playwright workers'` | failed (1 failing case) | `2026-10-27T16-59-11-096Z-p2995-r90394ebd75419cf9` | -| `2460d06a` | `--file test/repo/worker-builds.test.ts` | failed (4 failing cases) | `2026-10-27T17-01-33-418Z-p3544-rf0ed6ae6891b7bef` | -| `2460d06a` | `--file test/node/worker-build-lifetime.e2e.test.ts --grep 'vitest workers'` | failed (1 failing case) | `2026-10-27T17-02-51-359Z-p4091-r5570749d7be150b1` | -| `8bfd952f` | `--file test/repo/worker-builds.test.ts` | passed | `2026-10-27T17-04-17-350Z-p4640-rcf9e120c55c8be12` | -| `8bfd952f` | `--file test/node/worker-build-lifetime.e2e.test.ts` | passed | `2026-10-27T17-04-25-470Z-p4733-rbbb88b3e4252cfc0` | -| `8bfd952f` | `--file test/node/product-server-proxy.e2e.test.ts` | passed | `2026-10-27T17-04-54-699Z-p4840-r7088235135e9aca5` | -| `d43623ba` | `--file packages/web/test/e2e/theme-product-acceptance.test.ts --grep 'theme settings persist\|executable themes run only'` | failed (2 failing cases) | `2026-10-27T17-06-17-318Z-p5370-rdb44323ae2540294` | -| `d43623ba` | `--file packages/web/test/e2e/browser-demo-mode.test.ts` | failed (3 failing cases) | `2026-10-27T17-06-45-779Z-p5483-r1ecb1a123a8af3bd` | -| `a2fd5a1e` | `--file packages/web/test/e2e/theme-product-acceptance.test.ts --grep 'theme settings persist\|executable themes run only'` | failed (1 failing case) | `2026-10-27T17-08-32-289Z-p6045-r1adf8b65c466ea55` | -| `3156e303` | `--file packages/web/test/e2e/theme-product-acceptance.test.ts --grep 'theme settings persist'` | passed | `2026-10-27T17-10-55-998Z-p6619-r99415de847ed17ad` | -| `3156e303` | `--file packages/web/test/e2e/browser-demo-mode.test.ts` | passed | `2026-10-27T17-11-19-136Z-p6726-rf86e68498d027234` | -| `3156e303` | `--file packages/web/test/e2e/theme-product-acceptance.test.ts` | passed | `2026-10-27T17-11-40-111Z-p6829-r29aa92fa0c6cb29c` | -| `3156e303` | `--file test/repo/build-config-integrity.test.ts --grep 'canonical font preload build seam\|web bundle version stamp'` | passed | `2026-10-27T17-12-32-079Z-p7345-rfbd491c0ee9aed82` | -| `3156e303` | `--file packages/web/test/foundation-copy.test.ts` | passed | `2026-10-27T17-12-48-815Z-p7444-r94b6e463c178a6b4` | -| `3156e303` | `--file test/node/acp-home-context.test.ts` | passed | `2026-10-27T17-12-58-704Z-p7540-r84fc72db15ad1385` | -| `3156e303` | `--file test/node/update-reload.e2e.test.ts` | passed | `2026-10-27T17-13-13-727Z-p7640-rd72830577adb0018` | -| `3156e303` | `--file test/node/update-reload-guard.e2e.test.ts` | passed | `2026-10-27T17-13-31-883Z-p7741-r161ae1bc0a41232f` | -| `3156e303` | `--file packages/web/test/e2e/update-reload.spec.ts` | passed | `2026-10-27T17-14-11-154Z-p7844-r2c4c37ea91b77a0a` | -| `3156e303` | `--file packages/web/test/e2e/update-journey.spec.ts` | passed | `2026-10-27T17-14-29-358Z-p7946-r2c2aef1f919cbe7d` | -| `3156e303` | `--file packages/web/test/e2e/update-notification.01.spec.ts` | passed | `2026-10-27T17-15-05-017Z-p8039-rbee79ef58ccb9edd` | -| `3156e303` | `--file packages/web/test/e2e/update-notification.02.spec.ts` | passed | `2026-10-27T17-15-28-580Z-p8147-r1c56a1d06122b465` | -| `3156e303` | `--file packages/web/test/e2e/gate-boot-barrier.spec.ts` | passed | `2026-10-27T17-15-49-683Z-p8247-r828004637951bbdb` | -| `38976455` | `--file test/repo/worker-builds.test.ts` | passed | `2026-10-27T17-19-37-128Z-p9328-r8f26a6a3c4857187` | -| `38976455` | `--file test/node/product-server-proxy.e2e.test.ts` | incomplete | `2026-10-27T17-19-45-200Z-p9422-r9fa89f239159c80d` | -| `38976455` | `--file test/node/product-server-proxy.e2e.test.ts` | passed | `2026-10-27T17-25-51-595Z-p10092-r14dcdf8329a685aa` | - -Every completed green run above has zero recovered native retries, process leaks and infrastructure failures. One final proxy rerun stalled in the coordinator’s HTTPS `git fetch origin --prune`, before acquisition. After confirming the Git child’s PID, start time, parent and command, I stopped that child on the private coordinator. The canonical result is retained as **incomplete**, with unknown tests and one infrastructure failure; it is neither red nor green evidence. Repeating the identical file selection passed. The repeat used process-local Git low-speed limits (one byte/second for 60 seconds) to bound another network stall; no repository, runner timeout or retry default changed. - -## Request inventories and limits - -The red theme runs observed 254–255 distinct development-path records per application document. On the passing complete walks, the persistence page has 38 distinct request records, and the two executable pages have 27 and 26. The three demo walks have 25, 20 and 20. All six application documents request `/` and built `/assets/` JavaScript; none of their complete retained inventories contains a development path (`/packages/`, `/@vite/`, `/@id/`, `/@fs/` or `/node_modules/`). Inventories omit query values, fragments and headers, and deduplicate by path, resource type and main-frame membership. These are loading-path observations, not raw request counts, peak-memory measurements or incremental savings after PR #32. - -The full theme file on the private Chromium image passed its five Chromium cases and skipped its Firefox-only case. The optional Firefox project was unavailable on that image. This slice makes no new Firefox execution claim. The focused build-config selection covers its four builder/font-preload cases; unrelated CLI telemetry-build cases await the integrated gate. - -Full verification, the killed-worker cleanup backstop, renewable reservations through retrieval, abandoned-run recovery, scratch cleanup and bounded failure diagnostics remain pending under the later slices. The private coordinator and two workers remain allocated for that work. No pool drain, worker recreation, shared-worker experiment or production resource-behavior change was performed. - -Primitive's commit hook surfaced a truncated set of existing decisions; that was not a complete constraint check. No decision was overridden or reconciled. This slice implements the converged direction and establishes no new durable team policy. diff --git a/docs/working/test-harness-resource-cleanup-slice-2-review-r1.md b/docs/working/test-harness-resource-cleanup-slice-2-review-r1.md deleted file mode 100644 index 5169ff09..00000000 --- a/docs/working/test-harness-resource-cleanup-slice-2-review-r1.md +++ /dev/null @@ -1,57 +0,0 @@ -# Slice 2 implementation review, round 1: test-harness resource cleanup (PR #12, 296391c0) - -**Verdict: PASS.** Slice 2 delivers the diagnostics the reporting spec promises and no more: four Linux readings with units, source and time at start and end of every Blaxel surface execution, written before preparation and finalized after owned-process cleanup on both the planned and the targeted path; unknowns kept as reasons, never zero; a counter reset that cannot become a negative delta; kernel OOM evidence read only when a native attempt failed or evidence is incomplete, through a two-second, 128 KiB-capped `dmesg` window filtered to the exact interval, retaining only process identity and numeric fields; one writer that enforces 64 KiB of kernel lines, 128 KiB per sidecar and 8 MiB per shard attempt including receipts, with compact observations reserved before excerpts; links that name only retained files, relative to the run directory, kept across infrastructure retries; and a coordinator download that accepts only files matching the run, shard and attempt it dispatched. The three proof assertions assigned to this slice are discharged by tests at the declared shapes, with the declared substitutes and nothing else mocked, and the record's evidence checks out row by row. No production file changed. Three refinements and one observation follow, none blocking. - -- Reviewed: `git diff 036d0491 296391c0` in full (the new module and types; coordinator, planned worker, supervised command, reporting, provider normalization and outputs; three new tests, three fixtures, the combined-project seam extension; the proof regrades; the record); against the plan's slice 2, `^resource-diagnostics` and `^resource-diagnostic-bounds`, the three proof assertions, the testing policy and the reviewer checklist; plus the retained run directories and static logs in this worktree. -- Branch state: contains `origin/main` (`a8e73fbf`); the copied slice 1 review is identical to mine; the response applies my proof refinement, declines the optional cost cut with a stated reason, and adds the static-log note. -- I ran nothing. I read summaries, one downloaded sidecar and two logs with small scripts; nothing was built, tested or changed. - -## What the slice proves, against the proofs - -**`^reporting-t-resource-bounds` (contract, `test/repo/resource-diagnostics.test.ts`, five cases).** With authored probe readings, a controlled clock and an injectable write, in a real temporary root: a baseline file exists before finalization with `end.status: "missing"`; an `EACCES` probe is kept as `value: null` with its reason; a counter that goes 10 → 2 yields `oomDelta.reason: "counter-reset"`; a recorded Playwright retry fixture (`test/repo/fixtures/test-runner/playwright-retry.json`) is classified `failedAttempt: true, recovered: true` and triggers a kernel request for exactly the start/end interval; a secret planted in identity, probe errors and a kernel line never reaches the file or the warnings; all-`EPERM` probes leave every value unknown with `kernel.reason: "EPERM"`; 8,000 kernel lines truncate to at most 64 KiB inside a file of at most 128 KiB; eighty 3,000-process receipts saturate the attempt budget and a pending baseline still gets its compact final observation; an `ENOSPC` write warns and returns no link; and a whole-second read is filtered to the exact interval with untimestamped OOM lines counted as omitted. That is the contract's text, clause by clause. - -**`^reporting-t-resource-parent` (seam, `test/repo/test-runner-resource-parent.test.ts`, four cases, plus the combined-project case in `test-runner-shard-worker.test.ts`).** The real planned worker (`scripts/run-test-shard.mjs --plan`) and the real targeted supervisor (`scripts/test/supervised-command.mjs`) each run an authored Vitest fixture under installed Vitest with `retry: 2`. The pre-command (`prepare.mjs`) reads the results tree during preparation and records what it finds, which is how the test proves the baseline existed, with its end missing, before preparation; the fixture's `afterAll` records teardown time, and the final observation is later than it. Each of the four probes is checked for source and units, with real values only required to be non-negative or carry a reason. The attempt file shows `["failed","passed"]`, so the recovered retry is real. Kernel status must be `available` or `unavailable` with an attributable code; the generic `unavailable` that a malformed invocation produces is refused, which is exactly how this seam caught the installed util-linux parser rejecting the first three timestamp formats (`0db76dd5` through `056f9e21`). The exit fixture (`setup.mjs` exits 17 before any test) leaves `end: observed, nativeEvidence: missing`; the interrupt fixture `SIGKILL`s its own supervising parent and the baseline survives with `end.status: "missing", reason: "parent-did-not-finalize"`, with the owned handle disposed and no survivor. The combined Vitest seam now checks one sidecar per project member with the right identity and a `resultPath` that resolves to that member's split native report. Nothing is mocked; the only hook is the guarded registry module, used by the local child and, as the proof hooks now say, never forwarded to workers. - -**`^reporting-t-resource-links` (contract, `test/repo/resource-diagnostics-links.test.ts`).** Two authored infrastructure attempts with real sidecar files: the superseded attempt's file survives in the history, the final surface links both attempts' files and drops a missing one, the recovered flake and clean failure list come from the recorded native fixture, every link resolves relative to the run directory, and neither `summary.json`, the compact provider summary nor `events.ndjson` contains a sample value or a kernel sentinel while the sidecar still does. A baseline-only provider failure normalizes to `provider:blaxel`, `incomplete`, `testStatus: unknown`, zero surfaces run, with the baseline linked. - -**The implementation behind them.** `startResourceDiagnostics` writes the baseline at construction and rewrites the same file at `finish`, deciding the kernel request from exit code, incompleteness, native status and failed attempts (`scripts/test/resource-diagnostics.mjs:31-60`); `readLinuxResourceProbe` reads `MemAvailable` and `Shmem` from `/proc/meminfo` (kB to bytes), `statfs` on the effective temporary directory and `oom_kill` from `/proc/vmstat`, and runs `dmesg --kernel --time-format iso` over an enclosing whole-second UTC window with a 2 s timeout and 128 KiB buffer, mapping `ENOBUFS` to truncation and permission text to `EPERM`/`EACCES` (`:79-102`); `kernelLinesInInterval` and `cleanKernel` keep only timestamped lines carrying `Killed process ()` and numeric OOM fields (`:110-122`, `:145-163`); `writeResourceDiagnostic` whitelists fields, reserves 8 KiB per existing file, binary-searches the excerpt to the budget and writes atomically (`:166-202`). The planned worker starts one collector per task before pre-commands and finishes it in the `finally` after `supervisor.finish()`, marking incompleteness on unconfirmed cleanup, failed pre-commands, service failure or leaks; the supervised command does the same around its whole run and, for Vitest targets, wraps the config with the attempt reporter so recovered retries are real evidence there too. The coordinator exports run, shard and `-attempt-` identity to the worker, lists `.testshards/results/resources`, caps the listing, refuses any file over 128 KiB or beyond 8 MiB in total or whose identity is not this run, shard and attempt, and copies the rest under `artifacts/shard-/resources/` on both the success and the failure path. Aggregation keeps every attempt's links while taking the latest outcomes; `retainedResourcePaths` keeps only files that exist, by exact path. - -## The record, checked - -| Claim | Evidence | -| --- | --- | -| 25 runs, each at the named commit with the named result; zero recovered retries, leaks and infrastructure failures | All 25 summaries present; `git.commit`, outcome and failure counts agree with every row; `testsFlakyRecovered`, `processLeaks` and `infraFailures` are zero throughout. | -| Lint and type-check passed at `86bdf6da` and `0e82df7f` | `.test-runs/slice-2-static/86bdf6da.log` and `0e82df7f.log` show both commands clean (the latter after its two test runs). | -| An inspected download at `f72654e1` retained a 1,605-byte sidecar with provider run ID, shard 1, attempt 1, `unit:root`, real values and resolvable references | `artifacts/shard-1/resources/surface-….json` is 1,605 bytes; `identity.runId` equals `request.json`'s run ID; `attemptId` is `-shard-1-attempt-1`; `resultPath` and `attemptResultPath` resolve; `kernel: not-requested`; `oomDelta: 0`; `results.json` links the run-relative path. | -| The kernel-reader corrections were test-driven on the worker | Six red rows between `0db76dd5` and `b9e9cbf8`, each with the stated failing count; the probes added for diagnosis are gone from the final tree. | -| Default budgets, private pool, nothing on the shared host | Every coordinator line in the logs names `--pool rose-cleanup` and `--target-retries 0` for `unit:root`. | - -## Are the diagnostics as small as the proposal intends? - -Yes. One JSON file per surface execution of about 1.6 KB when nothing failed; a kernel excerpt only on failure or incomplete evidence, capped at 64 KiB; 128 KiB per file and 8 MiB per attempt, enforced at write and again at download; no sampler, process inventory, heap snapshot, archive or daemon; samples kept out of the timing stream and console. The one growth beyond the proposal's minimum is that passing surfaces also keep their start/end file (observation below). - -## Refinements (none blocking) - -1. **Import the provider constant instead of copying it.** `resource-diagnostics.mjs:22` enables diagnostics only when `TV_TEST_TIMING_PROVIDER` equals a literal that `timing-baseline.mjs:8` already exports as `BASELINE_TIMING_PROVIDER`. A renamed worker tier would then switch diagnostics off silently. Importing the constant, or gating on an explicit variable the shard script sets, removes the drift. -2. **The targeted config path assumes a repository-rooted cwd.** The shard script exports `TV_TARGET_CONFIG` as `/workspace/television/`, but the CLI hands the coordinator a config path relative to the surface's `cwd` (`scripts/test/cli.mjs:810-812`). Correct today, because no Vitest surface declares a `cwd`, and the targeted runs in the record exercised it; a package-rooted Vitest surface would get a wrong path. Join with the target cwd, or say the assumption in a comment. -3. **Treat the receipt shape as provisional.** `cleanRecord` already serializes `kind: "recovery"` with the fields the spec names, and the budget contract exercises it, but no recovery data exists yet. Slice 3 should reconcile the shape against what recovery actually knows rather than inherit it. - -**Observation.** Sidecars are retained for passing surfaces as well as failing ones. The spec's retention sentence lists the cases that must be retained and does not forbid this; the attempt budget bounds it; and the baseline on a passing run is the context the proposal wanted. Say the choice in the record so the integrated review does not read it as drift. - -## Reviewer checklist - -- **Proof coverage.** The three assertions are discharged by the cited tests at the declared shapes; the live acceptance stays pending for slice 3 as planned. -- **Mocking and boundaries.** Contracts use only the probe reader, clock and write substitutions the proof hooks declare; the seam and the combined-project case have none; the native fixture is the declared recorded one. -- **Honesty of evidence.** The record distinguishes placeholder red, test-defect red, reader-incompatibility red and green; it claims no OOM reproduction and no live download qualification. -- **Scope.** Runner scripts, tests, fixtures, proofs and working documents; no production, dependency, timeout or retry change; the lease/download race is untouched and stated as pending. -- **Baseline.** Green for its obligations; `^reporting-t-resource-live` pending; the plan's slice 2 baseline statement holds. - -## Evidence and limits - -- Read every changed file; cross-checked all 25 run summaries, the cited provider run's sidecar, links and request, and both static logs; traced the baseline and finalization points in both execution paths and the identity from shard script to download. -- Not established: whether `dmesg` returns lines on the private workers under a real failure (the final parent runs accepted either outcome with an attributable reason; the record's probe at `056f9e21` shows the reader exits 0 there); the diagnostics' behavior across worker reuse and recovery, which is slice 3's. - -## Housekeeping - -- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel beyond scripts that read files already on disk. `git fetch origin` updated remote-tracking refs. -- Primitive produced no output during this review, so I have nothing to report on its usefulness. diff --git a/docs/working/test-harness-resource-cleanup-slice-2.md b/docs/working/test-harness-resource-cleanup-slice-2.md deleted file mode 100644 index aade87cc..00000000 --- a/docs/working/test-harness-resource-cleanup-slice-2.md +++ /dev/null @@ -1,68 +0,0 @@ -# Test-harness cleanup: slice 2 evidence - -Owner: rose. Draft PR #12, branch `thopter/test-harness-resource-cleanup`. Derived from the converged [plan](test-harness-resource-cleanup-plan.md). This slice implements bounded resource observations and their report transport. Worker exclusion through retrieval, abandoned-run recovery, temporary-directory cleanup and the dedicated live acceptance remain assigned to slice 3. - -## Changes and review scope - -The shared collector runs in the planned task parent and targeted supervisor. It writes a baseline before surface preparation and finalizes after service and owned-process cleanup. Combined Vitest projects receive separate surface records referencing their split native reports and shared attempt recorder. Missing final observations remain explicit after abrupt parent death. Linux probes retain `MemAvailable` and `Shmem` from `/proc/meminfo`, free bytes from `statfs` on the effective temporary directory, and `oom_kill` from `/proc/vmstat`, with units, source and observation time. Missing probes remain unknown; a reset cannot become a negative OOM delta. - -The targeted Vitest path now uses the existing generated runner config and attempt recorder. This supplies actual failed-then-passed attempts to both reporting and diagnostic capture; final JSON alone cannot establish a recovered retry. Diagnostic decisions use the existing native normalizer. A failed native attempt, incomplete evidence or a nonzero exit requests bounded kernel evidence for the execution interval. The kernel reader has a two-second timeout and a 128 KiB read cap; it retains only relevant OOM identities and numeric fields, with at most 64 KiB of lines. It does not retain arbitrary kernel messages or arguments. - -One writer enforces 128 KiB per sidecar and 8 MiB per shard attempt, including recovery receipts. It reserves compact observations for already-created baselines before excerpts consume the budget. Receipt serialization is available for slice 3; this slice does not produce worker recovery receipts. Truncation, omitted data and unavailable readings are explicit. Probe/write failures warn without replacing the native outcome. Neither sample values nor kernel excerpts enter console summaries or timing events. - -The coordinator copies only bounded files matching the current provider run, shard and infrastructure-attempt identity. Diagnostic attempt identity includes the retry number because the current lock ID alone is reused across retries. Provider aggregation retains each attempt's files while using the latest task outcomes; final `resourceDiagnosticsPaths` entries are relative to the canonical run directory and name retained files only. Sidecars keep relative links to their native JSON and attempt NDJSON. Baselines recovered without a task summary remain provider evidence and cannot invent a selected-surface result. - -No production application file, dependency, timeout default or retry default changes. Lease/release behavior is not changed here. The present release-before-download race remains until slice 3; this checkpoint is not safe concurrent-worker-reuse qualification. - -## Placement, red/green and corrections - -All tests used committed, pushed revisions from `/workspace/television` on `tv-cleanup-coordinator-rose`, through the canonical Blaxel runner with private pool `rose-cleanup`. The two execution workers were `tv-testshard-x64-rose-cleanup-01` and `-02`. Nothing was tested, built, linted or type-checked on the shared development host, and no shared worker was touched. All outer selections are `unit:root` with the default zero native retries and two infrastructure retries. The authored nested fixtures deliberately retry to establish actual failed and passed attempt evidence. No isolated-GitHub lifecycle fixture ran. - -- `bda9b6cd` contains the three focused files and a minimal writer placeholder so failures reach assertions rather than missing imports. Four collector contracts failed on absent observations/bounds and an unretained link; two link contracts failed on absent path/history propagation; four real parent cases failed on absent baselines. The native fixtures reached their intended pass, early exit or interruption paths. These are red evidence, with no infrastructure failure or leaked process. -- `e4a094a7` adds diagnostic assertions to the existing real combined-project seam. It failed on missing paths after correctly recording one passing member and one recovered member. -- `24229bd3` implements the collector and transport. Collector contracts and the combined-project case passed. The link test reached its timing-event assertion and exposed an authored fixture typo (`id` instead of `runId`); the interruption test exposed that SIGTERM legitimately lets the supervisor finish and retain a final sample. These were test defects, not evidence that those production paths should change. -- `86bdf6da` corrects the timing fixture and makes the tiny preparation fixture SIGKILL its own parent, then exit itself. The parent test waits for that signal and disposes its owned handle. This proves missing final evidence without leaving a descendant. The budget contract also finalizes a second baseline after receipts saturate the attempt budget. All three focused files then passed, followed by complete reporting, provider and shard-worker files. Lint and type checking passed on the coordinator; the log is retained at `.test-runs/slice-2-static/86bdf6da.log`. -- `83e8c9f1` stops labelling every nonzero kernel-reader exit as a permission error. Permission reasons now require corresponding command evidence; other failures remain unknown. Collector and parent files passed. `f72654e1` distinguishes diagnostic infrastructure attempts and passed the provider file through real download. -- Final inspection found that accepting any unknown kernel-reader error could conceal a malformed invocation. The [util-linux calendar parser](https://github.com/util-linux/util-linux/blob/v2.39.3/lib/timeutils.c) does not accept the trailing `Z` of JavaScript ISO timestamps. The stricter real-reader assertion at `0db76dd5` failed in both planned and targeted cases with the generic `unavailable` reason. `cf4fb510` tried epoch seconds; the installed worker parser rejected those too. A narrowed diagnostic revision, `734a29ac`, retained `dmesg: invalid time value "@1793124854.151"` in the native assertion. `dcc0847c` tried fractional calendar bounds, which also failed. The narrowed `056f9e21` probe established util-linux 2.38.1 on the worker: whole-second calendar times returned status 0; both fractional calendar and epoch times were rejected. The final reader uses an enclosing whole-second UTC window, then filters the returned ISO timestamps to the exact observation interval. `b9e9cbf8` first adds a red contract for that filtering, including disclosed omission of OOM lines without timestamps. The temporary command probes are removed from the final seam. At `0e82df7f`, all five collector contracts and all four parent cases passed; lint and type checking also passed on the private coordinator. The final static log is retained at `.test-runs/slice-2-static/0e82df7f.log`. - -## Canonical run evidence - -Each selection below follows `npm test -- blaxel --pool rose-cleanup`. The orchestration-only `--run-dir-output` receipt does not change selection. Run directories exist under `/workspace/television/.test-runs/` on the coordinator and under `.test-runs/` in this worktree. Retained files include canonical summary/results/events, provider request/report/setup/shard/log and native JSON. Downloaded diagnostic files and their referenced native/attempt files are retained locally for the inspected runs. Every listed run has zero process leaks and zero infrastructure failures; outer passing runs have zero recovered retries. Nested retry evidence is checked by the real seams rather than counted as an outer flaky test. - -| Commit | Canonical selection | Result | Run-directory name | -| --- | --- | --- | --- | -| `bda9b6cd` | `--file test/repo/resource-diagnostics.test.ts` | failed (4 failing cases) | `2026-10-27T17-47-54-943Z-p10651-r3ac5350dc8a133e8` | -| `bda9b6cd` | `--file test/repo/resource-diagnostics-links.test.ts` | failed (2 failing cases) | `2026-10-27T17-48-04-531Z-p10744-r14c37ae4adc16ed0` | -| `bda9b6cd` | `--file test/repo/test-runner-resource-parent.test.ts` | failed (4 failing cases) | `2026-10-27T17-48-14-171Z-p10838-r3883e2af32ea2e2a` | -| `e4a094a7` | `--file test/repo/test-runner-shard-worker.test.ts --grep 'runs exact files from two Vitest projects in one process'` | failed (1 failing case) | `2026-10-27T17-59-15-407Z-p11411-rbae49973b433bf36` | -| `24229bd3` | `--file test/repo/resource-diagnostics.test.ts` | passed | `2026-10-27T17-59-49-606Z-p11918-r8fc8bc0fe0c4d379` | -| `24229bd3` | `--file test/repo/resource-diagnostics-links.test.ts` | failed (1 failing case) | `2026-10-27T17-59-57-947Z-p12010-r834336b8375e5b32` | -| `24229bd3` | `--file test/repo/test-runner-resource-parent.test.ts` | failed (1 failing case) | `2026-10-27T18-00-05-937Z-p12102-r5e012a44da71b58e` | -| `24229bd3` | `--file test/repo/test-runner-shard-worker.test.ts --grep 'runs exact files from two Vitest projects in one process'` | passed | `2026-10-27T18-00-13-973Z-p12194-r282952d31fc443ed` | -| `86bdf6da` | `--file test/repo/resource-diagnostics.test.ts` | passed | `2026-10-27T18-02-11-101Z-p12741-r418e495d464a8309` | -| `86bdf6da` | `--file test/repo/resource-diagnostics-links.test.ts` | passed | `2026-10-27T18-02-19-656Z-p12834-rfdb3a34bd4c461f6` | -| `86bdf6da` | `--file test/repo/test-runner-resource-parent.test.ts` | passed | `2026-10-27T18-02-28-420Z-p12926-r60d6db28b6e495cf` | -| `86bdf6da` | `--file test/repo/test-runner-reporting.test.ts` | passed | `2026-10-27T18-03-27-138Z-p13498-r6e3b77fbc4738fcd` | -| `86bdf6da` | `--file test/repo/test-runner-provider.test.ts` | passed | `2026-10-27T18-03-40-246Z-p13595-r2c5e2ec7f7595f57` | -| `86bdf6da` | `--file test/repo/test-runner-shard-worker.test.ts` | passed | `2026-10-27T18-03-48-465Z-p13687-r720be13bc8846ae1` | -| `83e8c9f1` | `--file test/repo/resource-diagnostics.test.ts` | passed | `2026-10-27T18-05-26-980Z-p14233-r1e947b2039364e32` | -| `83e8c9f1` | `--file test/repo/test-runner-resource-parent.test.ts` | passed | `2026-10-27T18-05-36-797Z-p14326-r2d359f1f9dcc8ed7` | -| `f72654e1` | `--file test/repo/test-runner-provider.test.ts` | passed | `2026-10-27T18-07-08-959Z-p14854-r758126113ffe4f59` | -| `0db76dd5` | `--file test/repo/test-runner-resource-parent.test.ts` | failed (2 failing cases) | `2026-10-27T18-10-50-295Z-p15426-reb1cb3404f6eb3af` | -| `cf4fb510` | `--file test/repo/test-runner-resource-parent.test.ts` | failed (2 failing cases) | `2026-10-27T18-11-37-879Z-p15942-rd69720c74d517af3` | -| `734a29ac` | `--file test/repo/test-runner-resource-parent.test.ts --grep 'targeted parent retains'` | failed (1 failing case) | `2026-10-27T18-14-10-644Z-p16480-rf4c2e4c1ed35a761` | -| `dcc0847c` | `--file test/repo/test-runner-resource-parent.test.ts` | failed (2 failing cases) | `2026-10-27T18-15-40-396Z-p17010-r22c8011ef51efab8` | -| `056f9e21` | `--file test/repo/test-runner-resource-parent.test.ts --grep 'targeted parent retains'` | failed (1 failing case) | `2026-10-27T18-17-26-697Z-p17541-rf5de936fb50934b4` | -| `b9e9cbf8` | `--file test/repo/resource-diagnostics.test.ts --grep 'restricts a whole-second'` | failed (1 failing case) | `2026-10-27T18-19-53-507Z-p18065-r37c3f452fa9ba8bf` | -| `0e82df7f` | `--file test/repo/resource-diagnostics.test.ts` | passed | `2026-10-27T18-20-55-154Z-p18587-r0f555091be0582c5` | -| `0e82df7f` | `--file test/repo/test-runner-resource-parent.test.ts` | passed | `2026-10-27T18-21-03-544Z-p18679-r62fcc5d3e0807a57` | - -## Proof coverage and limits - -The [reporting proof](../../proofs/arch/test-runner/reporting.md) now cites the five collector/interval contracts, two link/history contracts and four real parent cases, plus the existing combined-project seam. The contracts use exactly the declared probe/clock/write substitutions and recorded native fixture. The parent seams use real Linux probes, preparation, installed Vitest, its attempt recorder and teardown. The abrupt-parent case has no replacement reporter or invented native result. The whole reporting file also exercises the established native-recorder crossing. - -An inspected successful coordinator download at `f72654e1` retained a 1,605-byte sidecar with provider run ID, shard 1, infrastructure attempt 1, surface `unit:root`, real start/end values, and resolvable JSON/NDJSON references. The identity maps to `provider/blaxel/request.json`'s run ID; the outer canonical run retains that request. This is useful transport evidence, not the dedicated planned/targeted reuse or recovery acceptance. Start/end values are not peak usage, and unavailable kernel evidence cannot exclude OOM. No OOM or resource exhaustion was induced. - -`^reporting-t-resource-live` stays pending. Slice 3 must qualify download under held ownership, subsequent cleanup/reuse, abandoned-run receipts, persistent-worker retention and actual Chromium/Node temporary placement. Full verification, integrated review, human spec acceptance and the shared-pool inventory question also remain outstanding. The private coordinator and both workers remain allocated for the next slice. - -Primitive's commit hook surfaced a truncated set of existing decisions; it was not a complete constraint check. No decision was overridden or reconciled. This slice implements the converged direction and establishes no new durable team policy. diff --git a/docs/working/test-harness-resource-cleanup-slice-3-review-r1.md b/docs/working/test-harness-resource-cleanup-slice-3-review-r1.md deleted file mode 100644 index 53859c45..00000000 --- a/docs/working/test-harness-resource-cleanup-slice-3-review-r1.md +++ /dev/null @@ -1,70 +0,0 @@ -# Slice 3 implementation review, round 1: test-harness resource cleanup (PR #12, 2dc0e7d6) - -**Verdict: FAIL, one blocker.** The lifecycle itself is right and the evidence for it is real. One worker is held from acquisition through confirmed teardown and retained results; only the coordinator renews, every worker-local change runs under one short `flock` guard, dispatch is registered before the API request and entry is refused once ownership changes, and abandonment is one branch of ordinary acquisition that stops attributed processes by token and PID/start, audits, clears the known leftovers, discards undownloaded results and keeps a bounded receipt. Pool publication waits for verification and deletion reserves before unpublishing. The contracts and Linux seams prove this at their declared shapes with only the declared substitutes, and the live driver crossed every acceptance the proofs order on a fresh private pool, including both coordinator kills. The validation record is accurate: all 45 run directories it names exist with the stated revisions, outcomes and counts, the failed live attempts failed where it says they did, and the deliberately shortened interval was used only where the plan allows. The blocker is not in the protocol: the committed qualification is bound by literal name to the author's personal coordinator and pool, in the canonical CLI as well as in the driver, which leaves the five new acceptance citations pointing at a test nobody else can run and that the plan's own cleanup will orphan. Three refinements follow, one of them about the reliability Josh asked for. - -- Reviewed: `git diff 296391c0 2dc0e7d6` in full (the worker lease module and lifecycle helpers, the coordinator, the pool manager, the CLI forwarding and placement guard, the diagnostics and supervisor changes, three new tests, the driver and fixtures, five modified tests, the proof citations, the record); against the plan's slice 3, the Blaxel spec's Locking, recovery, pool-operations, Temporary files and Testing sections, the harness placement and registry specs, the testing policy and the reviewer checklist; plus the retained run directories, the eight `blaxel-live-*` attempt directories and the static logs in this worktree. -- Branch state: contains `origin/main` (`a8e73fbf`); the copied slice 2 review is identical to mine; all three slice 2 refinements are applied (`BASELINE_TIMING_PROVIDER` imported, the targeted config joined with its cwd, receipts carrying the real prior run, worker, deadline and process cleanup). -- I ran nothing. I read summaries, native reports, receipts, sidecars, logs and the live attempt records with small scripts; nothing was built, tested or changed. - -## What the slice proves, against the proofs - -**Lease decisions (`^blaxel-t-lease-decisions`).** `test/repo/blaxel-lease-contracts.test.ts` runs the real `workerLeaseOperation` in disposable roots with authored records and substituted clock, scan and signal hooks: an expired lease recovers through the same `stopOwners` call whether a token-bearing process exists or not, and an unrelated surviving owner holds the old identity (`:23-35`); a legacy record, an empty record and unreadable JSON each deny both acquisition and deletion while any owner is seen and admit them on a clean audit, an unexpired legacy TTL holds, and audit mode denies a held or expired worker without inspecting it and preserves residue through prepare and teardown (`:37-60`); a signal failure during recovery leaves the old record, while explicit deletion reserves an expired worker without a scan (`:62-68`). The seam adds the renewable default of 1,200,000 ms and `expiresAt: 0` (`blaxel-worker-lease.test.ts:21-29`) and old-identity renewal and release refusal (`:41-53`). - -**Coordinator ordering (`^blaxel-t-lease-order`).** `finishLease` is proven to settle writes, tear down, retry the download without re-executing anything, and release only after both succeed; a teardown error, a persistent download error or a lost assertion each prevent release (`:106-116`). Renewal keeps ticking while a download waits and stops on the first lost renewal (`:138-159`). An undispatched but prepared attempt downloads its produced file before releasing, through the real `prepare`/`teardown`/`release` operations with only the SDK read substituted (`:169-192`). The provider file holds the forwarding contract (`leaseRecoveryArgs` empty by default, the exact `cli.mjs:484` wiring) and the interrupted-report ordering with its updated finalization string. The seam shows a request that never registered denies teardown and release (`:88-89`). - -**Guarded registration (`^blaxel-t-worker-guard`).** The seam executes the real generated `flock … node -e` command under installed `bash`, `flock` and procfs: two concurrent acquisitions yield one owner; a killed guard holder frees the OS guard and the surviving lease is untouched (`:62-78`); a real `workerEntryCommand` delayed until after recovery exits non-zero without touching the checkout sentinel (`:80-96`); a real nested `SurfaceSupervisor` with a restricted child environment still carries the outer `TV_TEST_SHARD_OWNER` while replacing the surface token, recovery stops the registered tree including a child that ignores TERM, and an unrelated owner both survives and holds the worker until it is gone (`:98-130`). The only substitution is the fixture-token filter on the audit, as declared. - -**Cleanup (`^blaxel-t-scratch-cleanup`, `^blaxel-t-scratch-errors`).** The real cleaner removes exactly both legacy prefixes and both non-current scratch directories, keeps the current one, the unrelated sibling and a symlink's outside target, reports counts, bytes and free space, removes only the current directory in `onlyCurrent` mode, and refuses a symlinked root (`blaxel-lease-contracts.test.ts:70-86`); a device mismatch, an identity change between measurement and removal, and an `EIO` removal each leave the directory, an empty `removed` map and one error (`:88-104`). The seam's last case proves `attempt.json`, the receipt and the inventory survive the checkout reset and scratch is gone after teardown (`blaxel-worker-lease.test.ts:133-145`). - -**Pool mutation (`^blaxel-t-pool-order`, `^blaxel-pool-pagination`).** Publication happens only after wait, provision and verify; a failed verification never publishes; deletion reserves, unpublishes, then deletes, and a refused reservation touches nothing; a never-published worker is deleted without a reservation and a failed deletion is retried (`:118-135`, `:161-167`). Discovery now returns provisioning and unavailable records to the manager and only published `DEPLOYED` workers to the coordinator (`blaxel-pool-discovery.test.ts`). - -**Registry (`^registry-t-blaxel-harness-selection`).** Both surfaces are `kind: "experiment"`, `all` and `e2e-ci` select neither, the tag selects only the fixtures, and the driver's two selections name only the fixture surface (`test-runner-config.test.ts:247-256`). - -**Live acceptances (`^blaxel-t-retrieval`, `^blaxel-t-recovery-live`, `^blaxel-t-tmpdir-live`, `^blaxel-t-pool-live`, `^reporting-t-resource-live`).** The driver at `45202ee9` (`.test-runs/blaxel-live-YSZi2G/`, `completed.json` passed, `cleanup.json` exit 0) deleted and re-provisioned the pool, seeded both leftover prefixes, stale scratch, an unrelated `/tmp` file and a `/cache` file on each worker, held a planned two-shard run at the download rendezvous, observed `lastRenewal` advance on both workers, saw a competing planned run fail to acquire either worker and the manager refuse deletion of both (`held-delete.log`), confirmed the owner records and the held shard reports were unchanged, then released the gate. Each shard's uploaded script was inside its scratch, the `temporary-cleanup` phase recorded one directory per prefix, the lock and scratch were gone afterwards and both sentinels survived. The targeted run reused the workers. Both crash scenarios killed the real renewing coordinator (the PID in the owner record), waited past the 60-second deadline, and the next planned run recovered both workers: receipts of 990 and 983 bytes name the old lock IDs, their exact `lastRenewal` and `recoverAfter`, `cleanup: "confirmed"`, `discardedReports: true` and seven attributed processes each with SIGTERM and SIGKILL outcomes (`bash`, `node`, `dash`, `esbuild`); the quiet receipts are 491 bytes with empty process lists. Every child run reports the intended one failed and one recovered fixture, completed infrastructure and no leaks; failed-surface sidecars are 1,737–1,787 bytes with `kernel: available`, and the placement records show Node's directory and sixteen Chromium descriptors, four of them unlinked, under `/tmp/tv-testshards/r-…/tmp`, with Chromium 148.0.7778.96 from the documented cache path. The final manager deletion removed both workers, the listing was empty and a fresh dispatch failed on the absent pool. - -## Is recovery as simple and reliable as Josh asked? - -Simple, yes. The whole worker-side protocol is one serialized function with nine operations (`scripts/test/blaxel-worker-lease.mjs:31-219`); recovery is the single `valid && expired` branch of acquisition (`:124-128`) and reuses the same `stop` and `clean` that normal teardown uses, so active and quiet orphans need no separate path (the live quiet receipts show the same procedure finding nothing). The coordinator adds a ten-line renewal loop and a five-step finish order (`blaxel-lease-lifecycle.mjs:5-32`). Every uncertain state holds rather than guesses: an unregistered dispatch denies teardown (`:184`), a surviving or foreign owner denies acquisition and teardown (`:126`, `:129`, `:186`), a cleanup error leaves the old record, a lost renewal stops every further mutation (`run-blaxel-testshards.mjs:328`, `:446`, `:469`), and each of these resolves at the next acquisition after the interval, which is what Josh wanted: a crashed run holds a worker for twenty minutes, never forever. Reliable, with one qualification in refinement 1: the coordinator gives a lease up on the first failed renewal exec, which is safe but turns a transport hiccup into a held worker and a failed shard. - -## The record, checked - -| Claim | Evidence | -| --- | --- | -| 45 named runs at the named revisions with the named results | All present; `git.commit`, status and counts agree with every row, including the two intended reds (`1320c957` two failures; `06b07c0f` one failure under `--grep`), the two provider-file source failures at `49cfe0c3`, the nine incomplete runs, and the four at `720cef01`/`5b23a426` selected by `--grep`. | -| `12939310`: six seam cases passed on both workers; retrieval failed on a missing `attempt.json`; contracts never ran | `manual-worker-01/02/native-shard-1.json` each 6/6; `shard-1.log` shows the 404 for `attempt.json` after a confirmed teardown; the contracts run failed with "No worker was available". | -| Static checks at each revision | `e102e200` five type errors in the contracts file; `e59caa56` clean; `53d800fa` the missing `wait`; `c7482876`, `49cfe0c3`, `9d197e8f`, `d32457f7` and `45202ee9` clean, each log's Blaxel summaries naming that commit. | -| Driver at `5b23a426`: preflight missing `ss`, then three failed attempts of 70.9, 357.7 and 348.8 s, the first on the fresh-only ensure, the other two on the final listing | The preflight run is `infraFailures 1` with the `ss` message; the attempt file carries exactly those durations; the native messages are `expected 2 to be +0` at `:97` and twice `toHaveLength(0) but got 2`; the intermediate receipts list seven and eight processes. | -| `9d197e8f`: passed on the second attempt after the placement assertion at `:163`; one targeted child recovered an infrastructure failure after a 60-second lease was rejected at entry | Native JSON: one failure at `harness.test.ts:163:123`, pass in 340.9 s; the child run has `attempt-2/` and its first log shows four `lease lost or expired` denials and the teardown rejection. | -| `d32457f7`: passed on the second attempt after a rendezvous timeout in the quiet scenario | `live harness rendezvous timed out` at `:171`, then pass. | -| Final run first-attempt green in 4m 31s, eight child runs, pool deleted | Attempt 1 passed (270 s); summary 271 s; the eight child outcomes are as tabulated; `final-delete.log` shows both deletions. | -| Shortened interval only on deliberate runs; defaults otherwise; private pool only; `iproute2` only on the coordinator | `--lock-ttl-seconds 60` appears on the seam run the record names and, in the driver, only on the two `abandoned` runs (`harness.test.ts:41-43`, `:170`); the recovered leases show the 1,200 s default; every coordinator line names `--pool rose-cleanup`; the two iproute logs show the install on the coordinator. | - -Not verifiable here: the 146-sandbox inventory, the Primitive CLI failures, and anything about the workers beyond the downloaded files. - -## Blocker - -**The committed qualification is bound by name to one person's sandbox and pool.** `scripts/test/cli.mjs:870-873` refuses `experiment:blaxel-harness` unless `TV_BLAXEL_HARNESS_COORDINATOR` equals the literal `tv-cleanup-coordinator-rose`; `harness.test.ts:10-11` fixes the coordinator to the same literal and the pool to `rose-cleanup`, and `:15` and `:109` name the mode file and sentinels after the author. The placement spec (`test-runner.md:149`) and the proofs' shared live declaration say "a dedicated credentialed Blaxel coordinator sandbox outside the worker pool", and the plan keeps this one coordinator only through the integrated gate and then deletes it. Once it is gone, the five acceptance citations this slice adds point at a test that cannot run without editing the canonical runner and the test, and the runner carries a personal resource name. The literal protects nothing: anyone can export that string on a laptop, and the real guards are the ones already present, the sandbox-label check in the driver (`harness.test.ts:97-98`) and the shard-token check in both places. Fix: read the coordinator name and the pool from the environment (the existing variable plus one for the pool), require them non-empty, keep the label and token checks, derive the mode file and sentinel names from the pool, and run the driver once more on a fresh private pool so the citations name the test in its committed form. No lease, recovery, retry or pool-manager change is involved. - -## Refinements (none blocking) - -1. **Renewal gives up on the first transport error.** `renewLease` marks the lease lost on any rejection (`blaxel-lease-lifecycle.mjs:5-15`), and `leaseOperation` rejects the same way for a worker denial and for a failed or timed-out exec (`run-blaxel-testshards.mjs:328-341`). One Blaxel exec failure during a twenty-minute lease therefore fails the shard as infrastructure and leaves the worker held for the whole interval while the lease is still good. The record's one unexplained lapse, the 60-second lease rejected at entry in `2026-10-27T20-01-14-534Z…`, is the kind of event this sensitivity produces. Treat an explicit `ok: false` as lost at once and retry a transport failure on the next tick while the deadline allows; renewal is idempotent, so the retry is safe, and a real expiry still ends as a denial. One function changes; the ordering contract already covers the loss path. -2. **Pin the coordinator's per-profile default.** The 900-second `normal` value the spec review flagged is now 1200 at `run-blaxel-testshards.mjs:1205-1207`, but the removed `lockTtlSeconds` assertion was not replaced and nothing holds "independent of the timeout profile". One source line beside the forwarding contract. -3. **Optional: remove the checkout reset rather than survive it.** `scripts/test/blaxel-repository.mjs:36,40` still removes `.testshards/results` during checkout, which is why `prepare` copies its evidence into scratch and the shell (`run-blaxel-testshards.mjs:887-889`, after checkout and in the EXIT trap) and teardown (`blaxel-worker-lease.mjs:187-190`) copy it back. Under the lease, `prepare` already resets that tree, so the checkout reset protects nothing and costs three restore points and the failure at `12939310`. Dropping it is the simpler shape, but it touches the repository-selection script and its seam, so it is the owner's call and can wait for the integrated pass. - -## Reviewer checklist - -- **Proof coverage.** All ten slice obligations, the pagination extension and the registry contract cite tests that passed on the private pool; the five live acceptances cite one driver run that passed at `45202ee9` and must pass again after the blocker's change to the driver. -- **Mocking and boundaries.** Contracts substitute only the declared seams (clock, owner scan and signals, cleanup probes, SDK operations, lifecycle callbacks, one SDK read); the seam substitutes only the audit's token filter; the driver's only hook is the pre-download rendezvous, and ordinary runs do not wait there (`run-blaxel-testshards.mjs:1081-1090`). -- **Honesty of evidence.** Red, setup, infrastructure and green outcomes are distinguished; intentional fixture failures are labelled as the driver's assertions; the unexplained lapse is stated as unexplained; no memory, OOM or savings claim is made. -- **Scope.** Runner scripts, pool manager, tests, fixtures, registry entry, proofs and working documents; no production file; the `normal` profile's recovery interval moves from 900 to 1200 seconds, which the converged spec requires. -- **Baseline.** Green for the slice's obligations; the full `poc` gate at 36 shards, target reconciliation and integrated review remain, as the plan states. - -## Evidence and limits - -- Read every changed file and the full current lease module, lifecycle helpers, pool lifecycle, driver and fixtures; traced acquisition, prepare, dispatch, entry, teardown, download and release through the coordinator and the shell; cross-checked all 45 run summaries, four native driver reports and attempt files, the manual worker evidence, two recovered runs' receipts and sidecars, three placement and cleanup records, the live attempt's command logs and recovery evidence, and all static logs. -- Not established: behavior of `flock -w 20` under a recovery that exceeds twenty seconds (a competing acquisition then simply skips the worker, which is the safe outcome); the shared `poc` pool's response to held workers at 36 shards, which the integrated gate will show; the cause of the one shortened-lease lapse. - -## Housekeeping - -- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel beyond scripts that read files already on disk. `git fetch origin` updated remote-tracking refs. -- Primitive: the record reports two warnings (`dec_d81a353e`, `dec_8e3d38a8`) from a CLI that would not start, one recovered decision text repeating the superseded no-expiry lock directive, and partial commit-hook scans; the author correctly treated none as authority. In my session Primitive's start hook injected a repository-connection notice and a standing instruction to invoke `prim` for lasting decisions; a review makes none, so I did not, and no conflict report or decision reached me. It contributed nothing to this review. diff --git a/docs/working/test-harness-resource-cleanup-slice-3-review-r2.md b/docs/working/test-harness-resource-cleanup-slice-3-review-r2.md deleted file mode 100644 index d1c600e3..00000000 --- a/docs/working/test-harness-resource-cleanup-slice-3-review-r2.md +++ /dev/null @@ -1,62 +0,0 @@ -# Slice 3 implementation review, round 2: test-harness resource cleanup (PR #12, dc1d9f2e) - -**Verdict: PASS.** The round 1 blocker is resolved at its root: the canonical CLI now admits `experiment:blaxel-harness` only with a non-empty coordinator name and pool supplied by the caller, local execution and no inherited shard token; the driver takes both names from the same variables, keeps the real label check on the named sandbox, derives its leftover fixtures and sentinels from the pool, and reads a neutral per-worker mode file; no personal resource name remains anywhere under `scripts/`, `test/`, the registry, the proofs or the specs. The CLI decision has its own red-then-green acceptance against the real process, the proofs name it, a README states how the qualification is invoked and what it destroys, and the complete live driver ran again in its committed form on fresh private workers and passed on its first attempt. The lease, recovery, cleanup, pool-manager and coordinator code is byte-identical to the round 1 revision (`git diff --stat 2dc0e7d6 dc1d9f2e -- scripts/` touches only the four CLI lines), so everything round 1 established about the protocol stands, and I re-read the whole slice against the proofs rather than only the correction. Refinement 2 is applied; refinements 1 and 3 are declined or deferred with reasons I accept below. No blockers; two observations, neither requiring a change before the integrated review. - -- Reviewed: `git diff 2dc0e7d6 dc1d9f2e` in full (CLI guard, driver, fixture support, the two guardrail cases, the profile pin, the README, two proof files, the plan line, the response and the record); the current driver, fixtures, lease module, lifecycle helpers, pool lifecycle, coordinator and manager as they now stand; the plan's slice 3; the Blaxel, placement and registry specs; the testing policy and the reviewer checklist; plus the new run directories, the new live attempt directory and the three new static logs in this worktree. -- Branch state: contains `origin/main` (`a8e73fbf`); the copied round 1 review is identical to mine; `dc1d9f2e` changes only documents and proofs on top of the executable checkpoint `7b0213bb`. -- I ran nothing. I read summaries, native reports, receipts, sidecars, placement and cleanup records, manager logs and the live attempt records with small scripts; nothing was built, tested or changed. - -## The blocker, resolved - -`scripts/test/cli.mjs:870-874` refuses the driver surface unless the provider is `local`, `TV_TEST_SHARD_OWNER` is absent, and both `TV_BLAXEL_HARNESS_COORDINATOR` and `TV_BLAXEL_HARNESS_POOL` are non-blank; the message names the two variables. `harness.test.ts:10-11` reads the same two values, `:104` refuses to start without them or under a shard token, and `:105-106` still fetches the named sandbox through the SDK and requires it not to carry the worker label, which is the check that actually distinguishes a coordinator from a pool worker. The leftover fixtures, stale scratch and both sentinels are named from the pool (`:17-23`, seeded at `:117-123`, asserted at `:168-169`); the mode file is `/tmp/tv-blaxel-harness-mode.json` on both sides (`:17`, `fixtures/support.ts:7`), written by the driver to each fresh worker and gone with it, so no production environment forwarding was added to carry fixture input. `test/runner-fixtures/blaxel-harness/README.md` says where the driver runs, what it requires, that it deletes the named pool before and after, and that the coordinator is the caller's to dispose of. The plan's invocation line now passes the names as configuration and says they are this contribution's resources, not the driver's. - -**Placement acceptance (`^t-blaxel-harness-placement`, `test/repo/test-runner-guardrails.test.ts:740-768`).** The real CLI, with authored environment values and the existing guarded dry-run hook, exits 0 and plans `surfaces=experiment:blaxel-harness` when both names are set; a blank coordinator, a missing pool, an inherited shard owner and `blaxel` as provider each exit 2 with the placement message and no `[dry-run]` output. The proof says what this proves (the CLI decision) and what it does not (sandbox identity, which the live driver's label check and configured pool carry). Red at `6d505597`, where the real CLI still demanded the literal and returned 2 under the neutral names; green at `7b0213bb`. - -**Live qualification repeated (`.test-runs/blaxel-live-Py7UqI/`).** `identity.json` records the configured coordinator, the configured pool and revision `7b0213bb`; `completed.json` is `passed` with all eight run pointers; `cleanup.json` is exit 0. The driver deleted the earlier pool and provisioned two fresh workers (`initial-delete.log`, `ensure.log`), the held deletion was refused on both workers while the planned run waited at the download gate (`held-delete.log`), the competing run acquired neither, the final deletion removed both and a fresh dispatch failed on the empty pool. Both workers' `temporary-cleanup.json` on the planned run report exactly one directory per prefix and one stale scratch directory under the pool-derived names; the recovered runs carry two receipts each, now 1,061 bytes with the worker name in `previous.worker`, eight attributed processes with SIGTERM and SIGKILL outcomes in the active scenario and 491-byte receipts with empty process lists in the quiet one; the abandoned leases show the 60-second interval and the recovered leases the 1,200-second default; failed-surface sidecars are 1,737–1,787 bytes with `kernel: available`; the placement records show sixteen Chromium descriptors, four unlinked, under the attempt's `tmp/`, Chromium 148.0.7778.96. The outer run passed its single native attempt in 295.1 s (summary 296.2 s) with zero failed or recovered tests, leaks or infrastructure failures, and lint and type-check passed at the same revision. - -## Refinement dispositions - -1. **Renewal stops on the first failed exec: declined, accepted.** The response keeps the fail-closed behavior and states the cost plainly: one failed renewal fails the shard as infrastructure and holds the worker until ordinary recovery. The reasoning is sound as far as it goes (a failed exec gives no evidence either way, and a safe retry needs the last confirmed deadline and new failure coverage), the behavior is correct rather than merely tolerable, and the proof's lease-ordering contract already names it. This is an availability trade-off the owner may make. It will be measured for the first time at the 36-shard integrated gate, where the number of renewal execs is two orders of magnitude larger than in this slice's runs; see observation 1. -2. **Profile-independent default: applied.** `test-runner-provider.test.ts:34-37` requires `lockTtlSeconds: 1200` for `normal`, `cold` and `repair` in the coordinator source, and the proof's provider-file list names the assertion as a source check with the seam and driver carrying behavior. -3. **Checkout reset: deferred, accepted.** Optional in round 1; the reason given (it touches repository switching, whose spec requires live two-repository evidence) is the one I anticipated, and the preservation path it would replace is covered and passed. - -## The whole slice, re-read - -Nothing in the lease state machine (`scripts/test/blaxel-worker-lease.mjs`), the coordinator lifecycle (`blaxel-lease-lifecycle.mjs`, `run-blaxel-testshards.mjs`), the pool lifecycle or the manager changed, so round 1's findings on them hold: one serialized worker-side function with nine operations, recovery as the single expired branch of acquisition reusing normal teardown's stop, audit and clean, coordinator-only renewal, dispatch registered before the API request and entry refused after ownership changes, release only after confirmed teardown and retained results, fail-closed holds on unregistered dispatch, unclean audits, cleanup errors and lost renewal, and pool publication after verification with deletion reserving before unpublishing. The eleven focused obligations and the registry contract still cite tests that passed on the private pool at `45202ee9`, and the only test files touched since then (`test-runner-guardrails.test.ts`, `test-runner-provider.test.ts`) passed whole at `7b0213bb` (65 passed with the one existing GitHub-only skip; 43 passed). The five live acceptances and `^reporting-t-resource-live` now cite a driver that has run in its committed form. - -## The record, checked - -| Claim | Evidence | -| --- | --- | -| `6d505597` intended red under `--grep`; four green rows at `7b0213bb` with the stated counts | Red run `testsFailed 1`, infrastructure completed, naming the "accepts a configured…" case; green runs: 2 of 66 under the grep, the provider grep passed, 65 passed and 1 skipped whole, 43 passed whole; zero recovered retries, leaks and infrastructure failures throughout. | -| Lint and type-check at `7b0213bb` | `rose-slice3-r1-green.log` ends with both clean, after summaries naming that commit. | -| Live run first attempt, 296.197 s, zero failures, retries, leaks, infrastructure failures | Attempt 0 passed in 295,131 ms; `summary.json` 296,197 ms at `7b0213bb`; counts all zero. | -| Eight child runs with the tabulated outcomes | All eight present at `7b0213bb`: four `failed/completed` with one failure and one recovered, four `incomplete` (two refusals, two killed coordinators). | -| Pool deleted, re-provisioned fresh, deleted again and confirmed absent | `initial-delete.log`, `ensure.log` (both workers provisioned to `DEPLOYED`), `final-delete.log`; the empty-listing and absent-pool assertions are in the passing driver. | -| 60-second interval only on the two abandoned runs | Driver source `:49-51`, `:181`; recovery evidence shows the abandoned leases at 60 s and the recovered runs' leases at 1,200 s. | -| No shared host, no shared pool | Every coordinator line names `--pool rose-cleanup`; the driver ran under `local` on the coordinator. | - -Not verifiable here: the 144-sandbox inventory and the Primitive hook behavior the record describes. - -## Observations (none blocking) - -1. **Report renewal losses at the integrated gate.** The declined refinement means a transport failure on any of the several hundred renewal execs a 36-shard run makes will show up as an infrastructure-failed shard, a retry on another worker and one `poc` worker held for twenty minutes. That is acceptable and spec-conformant, but it is the first time the choice will be exercised at scale; the integrated record should say how many shards, if any, failed with "Lease renewal lost" so Josh can judge the trade-off on evidence rather than on this slice's small runs. -2. **The guardrail cases clear the shard token deliberately.** `TV_TEST_SHARD_OWNER: undefined` in both new cases is what lets the real CLI pass on a Blaxel worker, where the test process itself carries the outer token; the spawned CLI is a dry run that exits at once, and the runs report no leaks. Worth one comment in the test so the next reader does not take it for an accidental weakening of ownership attribution. - -## Reviewer checklist - -- **Proof coverage.** Every slice obligation cites a test that passed on the private pool at the revision it names; the placement acceptance is new, cited, and red-then-green; the live citations name the driver that ran at `7b0213bb`. -- **Mocking and boundaries.** Unchanged from round 1 for the protocol; the new acceptance substitutes only the declared dry-run hook and authored environment values and says so. -- **Honesty of evidence.** The record distinguishes the intended red, the green rows and the repeated qualification, keeps the original history under a renamed heading, and claims nothing about the declined refinement beyond its stated cost. -- **Scope.** Four CLI lines, the driver and fixture input, two tests, a README, proofs and working documents; no production file; no lease, retry, timeout or pool change. -- **Baseline.** Green for the slice's obligations; integrated review, target reconciliation, the `poc` gate at 36 shards and pre-merge docs preparation remain, as the plan and record state. - -## Evidence and limits - -- Read every changed file and the current driver and fixtures in full; scanned `scripts/`, `test/`, the registry, the proofs and the specs for the personal names; cross-checked the sixteen new run summaries, the native reports behind the whole-file counts, the live attempt's identity, completion, cleanup and manager logs, both recovery-evidence files, all receipts, sidecars, placement and cleanup records of the four completed child runs, and the three new static logs. -- Not established: anything about the private workers beyond the downloaded files; the renewal-loss rate under a full-size run. - -## Housekeeping - -- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel beyond scripts that read files already on disk. `git fetch origin` updated remote-tracking refs. -- Primitive: the record reports that its commit-hook scans again returned truncated results and no new constraint, and that the earlier warning remains unread as the supervisor directed. In my session the start hook's connection notice and `prim` instruction were the only Primitive activity; a review establishes no lasting decision, so I did not invoke it, and nothing from Primitive reached or shaped this review. diff --git a/docs/working/test-harness-resource-cleanup-slice-3.md b/docs/working/test-harness-resource-cleanup-slice-3.md deleted file mode 100644 index 54283204..00000000 --- a/docs/working/test-harness-resource-cleanup-slice-3.md +++ /dev/null @@ -1,398 +0,0 @@ -# Slice 3: exclusive Blaxel worker lifecycle - -Status: [review round 2](test-harness-resource-cleanup-slice-3-review-r2.md) passed at `dc1d9f2e`; slice 3 has converged. Integrated validation is recorded [separately](test-harness-resource-cleanup-integrated.md). -Final executable checkpoint: `7b0213bb`. The repeated live qualification passed on -its first native attempt in 4m 56s. The correction and evidence follow the original -qualification history below. -The converged plan and proofs govern the scope. Slice 2 passed at `296391c0`. -No tests, builds, lint, type checks, browsers or servers run on the shared host. -All executions below use pushed revisions on the private Blaxel coordinator and -`rose-cleanup` workers. No shared-pool worker experiments have been performed. - -## Implementation - -The worker-local lock command carries its implementation before checkout and uses -one short `flock` guard. Its record owns an attempt-specific scratch directory and -an outer process token. Only the coordinator renews; normal release follows owned -process teardown and retained evidence. Expired renewal uses the same cleanup -procedure for active and quiet abandoned work. Old timestamps, incomplete records, -unknown dispatch and unsafe cleanup remain fail-closed at the reviewed boundaries. -Pool publication follows runtime verification; deletion reserves eligible workers -before unpublishing them. A never-published worker keeps that state if deletion -fails, so a partial provisioning failure does not require an installed runtime to -retry deletion. - -The live driver and its two native fixture files are explicit experiment surfaces. -They use only the named private pool and retain default native and infrastructure -retries. Only the two coordinator-crash invocations shorten recovery to 60 seconds; -ordinary qualification and production runs use the 1200-second default. -The final qualification below establishes the planned and targeted paths, both -crash scenarios, temporary placement, diagnostics, and pool deletion. - -## Validation record - -| Revision | Command on Blaxel | Retained run / result | -| --- | --- | --- | -| `1320c957` | `npm test -- blaxel --pool rose-cleanup --file test/repo/blaxel-worker-lease.test.ts` | `.test-runs/2026-10-27T18-41-03-525Z-p19298-r7dc71e154247c385`: two intended assertion failures, infrastructure completed. | -| `2c0f5d6b` | Same selected file | `.test-runs/2026-10-27T18-50-52-346Z-p19842-r74788ac655aa50a6`: implementation failure, not green. Vitest rewrote serialized dynamic imports; cleanup also rejected regular overlay files as mount boundaries. | -| `e102e200` | `npm run lint`; `npm run type-check` | Lint passed; type check found contract-test typing errors. `.test-runs/slice-3-static/e102e200.log`. | -| `e59caa56` | Same static commands | Both passed. `.test-runs/slice-3-static/e59caa56.log`. | -| `12939310` | Lease seam, then lease contracts, each selected by file through the canonical Blaxel runner | `.test-runs/2026-10-27T19-11-24-927Z-p21647-r03d2ada7561f5f9a`: all six lease seam cases passed on both workers; canonical retrieval failed because checkout removed `attempt.json`. The run is infrastructure-incomplete, not a green checkpoint. Contracts were not reached. | - -The failed cleanup retained both private leases rather than admitting new work. -Read-only inspection established that the worker overlay exposes device 19 for -directories and 20 for regular files under the same mount. The corrected cleaner -uses actual mount records plus directory device boundaries. It still refuses -mounted paths and symlink roots. Earlier reports and available native files were -copied before the next acquisition can discard abandoned worker evidence. - -The coordinator shutdown path now waits for in-flight acquisitions, shares one -teardown operation with normal completion, and reports failed releases. Optional -resource sidecars use only the bounded warning-only transport; the generic evidence -inventory cannot turn a diagnostic read failure into a native failure or bypass -its byte budget. - -## Review refinements and limits - -Slice 2 refinements 1 and 2 are implemented: reuse the timing-provider constant and -resolve a targeted Vitest config relative to its surface cwd. Recovery receipts -carry the actual prior run/lock, worker, renewal/deadline and attributed process -cleanup, rather than assuming a coordinator's OS death can be proven remotely. -The final live qualification establishes receipt retention and Chromium temporary-file placement. - -Primitive's warning `dec_8e3d38a8` (check -`td7crdek1b1n4dhajyww5p174s8h23bq`) remains unread because its CLI failed to install -or start. The supervisor directed no further debugging and confirmed it is not an -authority over the converged specs and Josh's decisions. Commit-hook scans also -reported partial/truncated results; those are not a complete conflict check. -No new lasting team decision is being established by this implementation. - -The next failed run confirmed expiry recovery, process disuse and scratch deletion -on both private workers. Its retained worker records and native reports are in -`manual-worker-01/` and `manual-worker-02/` under that run directory. The checkout -helper resets `.testshards/results`, so initial identity/cleanup/receipt evidence -must survive in attempt scratch and be restored after checkout, on shell failure, -and by confirmed teardown after a killed shell. The correction adds that fixture -boundary without changing repository selection or checkout behavior. Those -leases remained held until their ordinary deadlines (about 19:31 UTC); no lock was -stolen or deadline edited. The subsequent green runs recovered them normally. - - -Read-only inventory at 19:29:39 UTC counted 146 sandboxes, all reported DEPLOYED, -including the two `rose-cleanup` workers and `tv-cleanup-coordinator-rose` outside -the pool. This does not establish active-work concurrency or an account-specific -quota; it does establish that the historical 75 figure is not a sandbox-count -limit here. Fresh live qualification replaces two owned workers after deletion, -without increasing the private worker count or touching shared workers. - -`c7482876` passed lint and type checking on the dedicated coordinator. Its next -lease run deliberately supplies `--lock-ttl-seconds 60` to qualify recovery from -the retained failed attempt; native and infrastructure retry budgets stay at their -defaults. The following contract run uses the ordinary recovery default. - -## Focused green checkpoint - -The checkout-reset correction passed the canonical runner at `c7482876`: -lease seam `.test-runs/2026-10-27T19-32-10-022Z-p22962-rfb177538dcaa4d24` -(seven cases, deliberate 60-second recovery interval), then contracts -`.test-runs/2026-10-27T19-32-20-996Z-p23059-r89282347b68d6c0c` -(nine cases, ordinary interval). Both completed on the first infrastructure -attempt with no test failures, recovered retries or leaks. Downloaded recovery -receipts survived the results reset, and the leases released after retrieval. -Lint and type checking passed at this revision and again at `49cfe0c3`. -The combined logs are in `.test-runs/slice-3-static/rose-slice3-green3.log` -and `rose-slice3-affected.log`. The earlier `53d800fa` type failure was a test -callback declaration requiring an unused SDK method; its log is -`rose-slice3-static3.log` in the same directory. - -At `49cfe0c3`, the seam and contracts passed again in -`2026-10-27T19-33-58-506Z-p23646-raaae2ad64b76b375` and -`2026-10-27T19-35-29-126Z-p23746-r01712028c5135ed1`. The provider file in -`2026-10-27T19-35-43-906Z-p23846-r3318f29853e73be5` exposed two source assertions -that still expected the old dispatch and interrupted-report wiring. The callback -behavior assertions passed. `720cef01` updated those two source assertions, -then ran the narrow pair before their whole file and the affected files below. -All listed runs used `npm test -- blaxel --pool rose-cleanup --file ` -with ordinary retry and recovery defaults; the first also selected the two names -with `--grep`. Full downloaded trees are under `.test-runs//`. - -| Revision | Selection | Canonical run | Result | -| --- | --- | --- | --- | -| `720cef01` | `test/repo/test-runner-provider.test.ts (two named cases)` | `2026-10-27T19-37-58-570Z-p24402-r4cdc7d5562c47eee` | passed; infrastructure completed | -| `720cef01` | `test/repo/test-runner-provider.test.ts` | `2026-10-27T19-38-09-966Z-p24499-r70f1f3eed2e449a5` | passed; infrastructure completed | -| `720cef01` | `test/repo/blaxel-pool-discovery.test.ts` | `2026-10-27T19-38-20-913Z-p24595-r84122d6e0b62dd4c` | passed; infrastructure completed | -| `720cef01` | `test/repo/blaxel-stale-owner-audit.test.ts` | `2026-10-27T19-39-40-343Z-p24692-rcb3f084f0f0c7c95` | passed; infrastructure completed | -| `720cef01` | `test/repo/test-runner-config.test.ts` | `2026-10-27T19-39-51-507Z-p24788-r2f67497f4e209dab` | passed; infrastructure completed | -| `720cef01` | `test/repo/resource-diagnostics.test.ts` | `2026-10-27T19-40-02-197Z-p24884-r776a0c6fb3273936` | passed; infrastructure completed | -| `720cef01` | `test/repo/resource-diagnostics-links.test.ts` | `2026-10-27T19-42-16-320Z-p24982-rf5458848d49b258a` | passed; infrastructure completed | -| `720cef01` | `test/repo/test-runner-resource-parent.test.ts` | `2026-10-27T19-42-28-510Z-p25079-r8d56b8721e4a08db` | passed; infrastructure completed | -| `720cef01` | `test/repo/test-runner-shard-worker.test.ts` | `2026-10-27T19-42-41-867Z-p25175-rf392193489fc6ad8` | passed; infrastructure completed | -| `720cef01` | `test/node/owned-process.e2e.test.ts` | `2026-10-27T19-43-03-198Z-p25283-r3d5b7ad0c998a6d0` | passed; infrastructure completed | - -Every run in this table has zero failed tests, recovered retries, process leaks and infrastructure failures. - -At `5b23a426`, the delayed-entry case additionally checks that an unregistered -request cannot release after teardown; the audit case verifies that available-worker -audit preparation/teardown preserves legacy residue and non-current scratch. -Each new assertion passed narrowly, followed by both complete files: - -| Selection | Canonical run | Result | -| --- | --- | --- | -| `test/repo/blaxel-worker-lease.test.ts (named case)` | `2026-10-27T19-43-45-072Z-p25796-r8622f05d670cea64` | passed; infrastructure completed | -| `test/repo/blaxel-lease-contracts.test.ts (named case)` | `2026-10-27T19-43-56-561Z-p25892-rd9b8c692ec75d21a` | passed; infrastructure completed | -| `test/repo/blaxel-worker-lease.test.ts` | `2026-10-27T19-44-07-741Z-p25987-rbb26c267c14b8a41` | passed; infrastructure completed | -| `test/repo/blaxel-lease-contracts.test.ts` | `2026-10-27T19-44-18-970Z-p26082-r36dc8f04d41f53f9` | passed; infrastructure completed | - -All four completed on the first infrastructure attempt, with no failures, recovered retries or leaks. - -## Live qualification and intermediate failures - -The first driver invocation at `5b23a426` stopped at coordinator preflight: -`2026-10-27T19-45-11-581Z-p26606-r3f51d29611472c9b` records missing `ss`. -Installed `iproute2` on `tv-cleanup-coordinator-rose` only; no worker changed. -The next canonical invocation uses the same pushed revision and normal retry -budget. Its first native attempt (`blaxel-live-EQZRnS`) found that Blaxel accepts -sandbox deletion before listing stops returning the deleted workers. The manager -had removed dispatch eligibility before its accepted deletes; fresh-only ensure -correctly refused the still-visible pool. Its final cleanup completed. SDK 0.3.24's -`SandboxInstance.delete` awaits the delete API response, not absence from listing. -The driver correction at `9d197e8f` polls for confirmed absence after a successful -canonical delete before recreating or claiming the pool gone. No lease protocol, -retry default, or provider configuration changes for this correction. - -The second native attempt (`blaxel-live-UjVPFN`) crossed planned/targeted retrieval, -competition and held deletion, actual TMPDIR placement, and both coordinator-kill -scenarios before reaching the immediate final-deletion listing check. Its retained -planned, targeted, active-recovered and quiet-recovered runs respectively are: - -- `2026-10-27T19-47-58-353Z-p27645-r7a1cb06d08f363aa` -- `2026-10-27T19-48-57-858Z-p27882-r0bc76a565de0ad6d` -- `2026-10-27T19-50-34-628Z-p28106-r6bde140566d56c43` -- `2026-10-27T19-51-51-273Z-p28338-r1db5ab714ba682ea` - -Each reports the intentional one failed fixture and one recovered native fixture, -completed infrastructure and no process leaks. The active receipt lists seven -and eight attributed processes on the two workers, including SIGKILL for surviving -Node processes. Quiet receipts have empty process lists. Failed-surface sidecars -are about 1.8 KiB; kernel access was available with no matching lines during the -observed intervals. That does not establish the absence of an OOM outside those -intervals. Chromium `148.0.7778.96` from -`/home/playwright/.cache/ms-playwright/chromium-1208/chrome-linux64/chrome` -held sixteen observed descriptors under the attempt TMPDIR, including deleted -`.org.chromium.Chromium.*` files. These are placement observations, not a memory -pressure reproduction or measured savings claim. This intermediate driver revision -is not the final green qualification. - -The canonical driver run at `5b23a426`, -`2026-10-27T19-46-15-645Z-p27336-r76172043306620d7`, completed infrastructure but -failed all three native attempts. Its attempt recorder shows 70.9 s, 357.7 s and -348.8 s. The first failed the fresh-only ensure assertion; the other two reached -the final delete then observed two workers still listed. No other assertion failed. -All three attempt directories (`blaxel-live-EQZRnS`, `blaxel-live-UjVPFN`, -`blaxel-live-wQ4JDh`) retain their logs and successful final cleanup records. -Both full attempts' child run trees and the outer native attempt report are -copied under `.test-runs/`. Default retries were allowed to finish; no revision -changed under this invocation. The corrected one-case driver runs at `9d197e8f`. - -At `9d197e8f`, the first driver attempt (`blaxel-live-pyyGvj`) reached a targeted -run that recovered an infrastructure failure: -`2026-10-27T20-01-14-534Z-p30344-r4b185c22b003ef58`. Attempt 1's remote entry and -teardown rejected a lost/expired 60-second lease before native evidence existed; -its exact renewal/transport timing was not retained and the cause of that lapse -is not established. Default infrastructure retry ran on the other worker and -retained the expected native failed/recovered outcomes under `attempt-2/`. -The driver still looked for `temporary-placement.json` outside that attempt -subdirectory. `d32457f7` derives artifact roots from the retained shard output -instead. It also corrects the live driver's overly broad 60-second override: -only deliberate abandoned-run invocations use it, as the plan requires; ordinary -retrieval and reuse use the production 1200-second default. No production lease -interval or retry budget changed. - -The `9d197e8f` outer run -`2026-10-27T20-00-04-460Z-p29961-r0fbeb800a4123ebe` passed on its second native -attempt (8m 39s total), followed by clean lint and type checking. Its first native -failure is exactly the placement-artifact existence assertion at line 163; its -second attempt (`blaxel-live-7Frvi5`) passed every live assertion, observed both -private workers disappear, and retained `cleanup.json` with exit 0. The outer -result correctly reports one recovered driver attempt. The complete outer report, -both live attempt directories and all child runs are retained. The static output -is `.test-runs/slice-3-static/rose-slice3-live3.log`. - -The `d32457f7` driver run, -`2026-10-27T20-09-47-242Z-p31959-reebae862cce797b7`, passed on its second native -attempt, then passed lint and type checking. The first failure was the driver's -120-second download-rendezvous wait in the quiet scenario, not a native fixture -assertion; both gate files eventually existed. The completed retry is -`blaxel-live-9zKaAr`, with passed completion and exit-0 cleanup, and all child -reports retained. The final driver allows 600 seconds to reach that rendezvous, -covering the runner's 480-second cold execution allowance plus teardown. This -changes neither a native retry budget nor a production worker timeout. Its static -log is `.test-runs/slice-3-static/rose-slice3-live4.log`. - -## Preparation without dispatch - -A final source pass found that `executionRequested: false` was being used as a -proxy for “no evidence produced.” Preparation can already have produced identity, -cleanup and recovery files before dispatch fails. At `06b07c0f`, the unchanged -retention implementation was extracted into the existing lease-lifecycle module -and a focused contract crossed real worker preparation/teardown and local writes, -substituting only the SDK read transport. Canonical run -`2026-10-27T20-25-30-716Z-p34389-r43c8aec32b92d34f` failed at release because the -prepared file had not been downloaded; infrastructure completed and the missing -file assertion was the intended red. - -`45202ee9` tracks whether preparation was requested and stops treating that state -as evidence-free merely because execution was not requested. Available artifacts -are also downloaded before propagating a required log-read failure, so missing -logs cannot suppress other retained evidence. Unknown preparation or retrieval -still holds the lease. No new worker state or recovery branch was added. - -The retention correction passed narrowly, then the full contracts, provider and -stale-owner files passed at `45202ee9`, with no native failures, recovered retries, -leaks or infrastructure failures. Lint and type checking also passed; the combined -log is `.test-runs/slice-3-static/rose-slice3-retention-green.log`. - -| Selection | Canonical run | -| --- | --- | -| `test/repo/blaxel-lease-contracts.test.ts (named case)` | `2026-10-27T20-28-25-923Z-p34956-r7a8e2fe5488f771c` | -| `test/repo/blaxel-lease-contracts.test.ts` | `2026-10-27T20-29-15-006Z-p35077-rc75cd93e2b864b85` | -| `test/repo/test-runner-provider.test.ts` | `2026-10-27T20-29-27-245Z-p35174-r098f4b0d3cd1cad7` | -| `test/repo/blaxel-stale-owner-audit.test.ts` | `2026-10-27T20-29-39-395Z-p35271-r3dc4727db3aa7390` | - -## Initial qualification and handoff - -At `45202ee9`, `npm test -- local --surface experiment:blaxel-harness` on -`tv-cleanup-coordinator-rose` passed on its first native attempt in 271.207 seconds: -`2026-10-27T20-31-21-143Z-p35862-rde268122d6db9e34`. The outer driver reports zero -failed or recovered tests, process leaks and infrastructure failures. It kept the -ordinary native retry budget of two and infrastructure budget of two. Ordinary -child runs used the default 1200-second recovery interval; only the two deliberately -abandoned runs supplied `--lock-ttl-seconds 60`. - -The complete live attempt is `.test-runs/blaxel-live-YSZi2G/`. Its completion and -cleanup records show success and exit 0. Every pointer below resolves to a fully -copied local run tree under `.test-runs/`; the two deliberately killed coordinators -truthfully remain incomplete. The fixtures intentionally yield one failed native -case and one recovered case in each completed child run. These expected native -failures are assertions of the passing driver, not unaddressed suite failures. - -| Scenario | Canonical child run | Expected result | -| --- | --- | --- | -| planned transfer and retrieval | `2026-10-27T20-31-43-447Z-p36010-r102a02d53fa2b20e` | one deliberate native failure and one recovered case; infrastructure completed | -| targeted retrieval and reuse | `2026-10-27T20-32-52-337Z-p36243-r7d1d01b2f31bd9ae` | one deliberate native failure and one recovered case; infrastructure completed | -| held-capacity refusal | `2026-10-27T20-32-42-262Z-p36128-r7e9debd7b5412984` | incomplete; no passing selected surface | -| active coordinator killed | `2026-10-27T20-33-03-214Z-p36326-r50763ec34c56268e` | incomplete; no passing selected surface | -| active orphan recovered | `2026-10-27T20-34-11-856Z-p36450-rfdddc02ece515dce` | one deliberate native failure and one recovered case; infrastructure completed | -| quiet coordinator killed | `2026-10-27T20-34-25-068Z-p36548-r02e5eeb7df18d744` | incomplete; no passing selected surface | -| quiet orphan recovered | `2026-10-27T20-35-28-816Z-p36690-r7eb081f69c2f7b8b` | one deliberate native failure and one recovered case; infrastructure completed | -| dispatch refused after deletion | `2026-10-27T20-35-48-311Z-p36815-r3d7413d5032dbe88` | incomplete; no passing selected surface | - -The driver observed renewal while downloads were held, refusal of competing -acquisition and deletion, unchanged held reports, correct report/native-attempt -links after scratch removal, both known-prefix removals and stale-scratch removal, -and preservation of unrelated `/tmp` and persistent-cache sentinels. Real Node -paths and Chromium descriptors (including unlinked files) were under the shard's -TMPDIR. Active recovery signalled attributed processes; quiet recovery used the -same procedure with no survivors. Both incoming runs retained the old identity, -renewal/deadline, cleanup outcome and discarded-report receipts. - -The final pool manager deleted `tv-testshard-x64-rose-cleanup-01` and `-02`, waited -for confirmed absence, and a fresh canonical dispatch failed because the pool was -empty. The private coordinator remains allocated with its evidence for the later -integrated gate. No shared worker was used or changed during qualification. - -The implementation and proof citations are ready for independent review, not for -merge. Slice review, integrated-result review, target reconciliation, the ordinary -full Blaxel verification gate, pre-merge docs preparation and Josh's human spec/PR -review remain. This evidence establishes lifecycle behavior, not peak memory use, -measured suite-wide savings, a new exhaustion reproduction or provider latency -bounds. The earlier shortened-lease expiry's precise timing remains unknown and -is recorded above. No production behavior, default retry budget, browser version -or shard count was changed. Primitive's unread warning remains recorded above; -its CLI was not debugged further. - -## Slice review round 1 correction - -The [review](test-harness-resource-cleanup-slice-3-review-r1.md) found one blocker: -the live driver and CLI required personal resource names despite the general -dedicated-coordinator contract. The [response](test-harness-resource-cleanup-review-response.md#Slice 3 round 1: reusable qualification) -records the correction and all three refinement dispositions. The caller now -supplies coordinator and private-pool names through environment variables; both -are required, with the existing label, provider and shard-owner checks preserved. -Pool-derived sentinels and a neutral worker-local mode file remove the personal -fixture dependency. The harness README documents invocation and destructive pool -ownership. The lease/recovery protocol is unchanged. - -Read-only inventory at 21:03:58 UTC counted 144 DEPLOYED sandboxes, including the -dedicated coordinator and neither private worker. At pushed `6d505597`, the canonical -pool manager provisioned the two `rose-cleanup` workers. No shared pool was changed. - -All focused commands below ran on the private coordinator with -`npm test -- blaxel --pool rose-cleanup --file `, adding `--grep` only as -shown. Unit and infrastructure retry budgets retained their defaults. Every run -tree is copied under `.test-runs/` in this worktree. - -| Revision | Selection | Canonical run | Result | -| --- | --- | --- | --- | -| `6d505597` | `test/repo/test-runner-guardrails.test.ts --grep "configured Blaxel harness coordinator"` | `2026-10-27T21-04-23-673Z-p37437-r2770a6d409ac3de3` | intended red: real CLI returned 2 instead of 0 with non-personal fixture names; native assertion failed and infrastructure completed | -| `7b0213bb` | `test/repo/test-runner-guardrails.test.ts --grep "Blaxel harness"` | `2026-10-27T21-06-11-968Z-p37987-r702e58cd23f76853` | both configuration and refusal cases passed | -| `7b0213bb` | `test/repo/test-runner-provider.test.ts --grep "explicitly supplied recovery interval"` | `2026-10-27T21-06-43-017Z-p38102-rb266ce5fc5f4db0e` | explicit/default forwarding and all three 1200-second profile assertions passed | -| `7b0213bb` | whole `test/repo/test-runner-guardrails.test.ts` | `2026-10-27T21-06-55-197Z-p38199-rf555298cb899a967` | 65 passed; one existing skipped GitHub-only case | -| `7b0213bb` | whole `test/repo/test-runner-provider.test.ts` | `2026-10-27T21-07-24-246Z-p38301-rebd40cb7ccb5bc87` | 43 passed | - -All four green runs completed without failed tests, recovered retries, process -leaks or infrastructure failures. `npm run lint` and `npm run type-check` then -passed on the dedicated coordinator at the same revision. Combined remote logs -are `.test-runs/slice-3-static/rose-slice3-r1-red.log` and -`rose-slice3-r1-green.log`. - -The earlier Primitive warning remains unread as directed. This correction follows -the existing placement contract and changes no lasting decision. Commit-hook -scans again returned existing decisions with truncated coverage; they supplied no -new actionable constraint and are not a complete conflict check. No reconciliation -or further CLI debugging was attempted. - -### Repeated live qualification - -At pushed `7b0213bb`, the dedicated coordinator ran: - -```sh -TV_BLAXEL_HARNESS_COORDINATOR=tv-cleanup-coordinator-rose \ -TV_BLAXEL_HARNESS_POOL=rose-cleanup \ -npm test -- local --surface experiment:blaxel-harness -``` - -Canonical run `2026-10-27T21-08-38-267Z-p38866-r679cddb899712621` passed on its -first native attempt in 296.197 seconds, with zero failed or recovered driver -tests, process leaks and infrastructure failures. The retained live attempt is -`.test-runs/blaxel-live-Py7UqI/`; its identity records the configured coordinator, -pool and exact revision, `completed.json` says passed, and `cleanup.json` records -exit 0. The driver retained the default two native retries and two infrastructure -retries. Only its two deliberate abandoned runs used the 60-second interval. - -| Scenario | Canonical child run | Result | -| --- | --- | --- | -| planned retrieval | `2026-10-27T21-09-11-096Z-p39023-r7c42878bb3edd7a5` | expected failed/recovered native fixtures; infrastructure completed | -| held-capacity refusal | `2026-10-27T21-09-44-524Z-p39141-r01321059d8eb93e8` | expected incomplete refusal | -| targeted reuse | `2026-10-27T21-10-33-789Z-p39277-r92cce732e13126b3` | expected failed/recovered native fixtures; infrastructure completed | -| active coordinator killed | `2026-10-27T21-10-49-716Z-p39365-rda3ab840cf51883d` | expected incomplete run | -| active recovery | `2026-10-27T21-11-53-508Z-p39482-r6e6a876c57f088ee` | expected failed/recovered native fixtures; infrastructure completed | -| quiet coordinator killed | `2026-10-27T21-12-07-353Z-p39581-re263e8075af61eb2` | expected incomplete run | -| quiet recovery | `2026-10-27T21-13-11-418Z-p39676-rfa6c95cd577a4d4e` | expected failed/recovered native fixtures; infrastructure completed | -| absent-pool refusal | `2026-10-27T21-13-30-318Z-p39800-ra32edd1d26498fce` | expected incomplete refusal | - -The driver deleted the earlier private pool after its focused reports were retained, -provisioned fresh workers, and repeated every live acceptance. Pool-derived residue -was removed, unrelated sentinels and caches survived, and the neutral mode input -exercised both recovery scenarios. Both private workers were deleted and confirmed -absent at completion. The entire outer run, attempt directory and eight child run -trees are retained locally; the combined log is -`.test-runs/slice-3-static/rose-slice3-r1-live.log`. - -The source correction and proof/document updates are ready for independent -re-review. The dedicated coordinator remains for the later integrated gate. -No tests, builds or static checks ran on the shared host; no shared worker was -changed. Integrated review, target reconciliation, full verification and PR -readiness remain pending. The availability tradeoff on renewal errors and optional -checkout simplification remain as assessed in the review response, without claims -that those refinements were implemented. diff --git a/docs/working/test-harness-resource-cleanup-spec-review-r1.md b/docs/working/test-harness-resource-cleanup-spec-review-r1.md deleted file mode 100644 index 1f628fd8..00000000 --- a/docs/working/test-harness-resource-cleanup-spec-review-r1.md +++ /dev/null @@ -1,65 +0,0 @@ -# Spec review, round 1: test-harness resource cleanup (PR #12, cd8aa453) - -**Verdict: PASS.** Every decision in the converged proposal is in a spec, in the owner the proposal named, and the affected tree reads consistently: the Blaxel spec's lifecycle, the reporting contract, the shared execution model, the runner's owner rules and the testing policy agree with each other and with the unchanged specs around them. Nothing in another spec still promises the mechanism this change replaces. Eight refinements follow; the first two are worth making before Josh reads the deltas, and the rest are trims that keep the specs as simple as the proposal intends. - -- Reviewed: `git diff origin/main...cd8aa453 -- specs` (five specs and the regenerated index), against the proposal and response at `cd8aa453`, the reviewer checklist, main's rewritten spec policy, the unchanged specs and proofs that cite the replaced mechanism, and the code the specs describe. -- Branch state: the branch contains `origin/main` (`a8e73fbf`); the merge `74d22941` changes nothing outside `docs/working/` relative to main and nothing in `docs/working/` relative to the branch. The copied round 3 review is identical to mine. -- I ran nothing. Every check is a read of files in the checkout or of git history. - -## The proposal's decisions, located - -| Decision | Spec | Checked | -| --- | --- | --- | -| One lease through teardown and download; shell never unlocks | `blaxel-testshards.md:45-47` `^blaxel-release` | Matches proposal order 1-5. | -| Coordinator-only renewal; 20-minute interval via `--lock-ttl-seconds`, independent of the execution timeout | `:41`, options table `:126`, profiles `:135` | 1200 s and "20 minutes" agree; the old TTL rows are gone. | -| Automatic recovery: one sequence for quiet and active leftovers, kill by token, discard orphan reports, receipt | `:51-57` `^blaxel-recovery`; receipt contract `reporting.md:224` | Receipt shares the 8 MiB attempt budget; no separate cap. | -| Attempt owner token preserved beside the surface token; registration before checkout | `:43`; `test-runner.md:222` | Consistent with the one-variable token the supervisor replaces today (`scripts/test/surface-supervisor.mjs:62`). | -| Legacy records: `expiresAt: 0`, respect TTL, skip live legacy work | `:61` | Matches the old reclaim test (`run-blaxel-testshards.mjs:406`) and the matching-ID trap (`:773-775`). | -| Deletion may destroy an expired worker that failed cleanup; dispatch eligibility removed first | `:63` | The SDK can change labels on a live sandbox (`@blaxel/core` `SandboxInstance.updateMetadata`). | -| Labels published after provisioning; no silent reuse; `ensure` does not refill | `:33` | Same SDK call. | -| `--audit-stale-owners` reserves without recovery | `:65`, table `:133` | | -| Per-attempt scratch, `TMPDIR`, exact two-prefix cleanup, every non-current attempt directory | `:69-80` `^blaxel-scratch`, `^blaxel-known-leftovers` | Caches, lock, guard, uploads and X11 preserved. | -| Minimum diagnostics, retention rules, 64 KiB / 128 KiB / 8 MiB bounds, honesty about what the numbers mean | `reporting.md:214-226` `^resource-diagnostics`, `^resource-diagnostic-bounds`; fields in `reporting.md:85` and `sharded-execution.md:255` | | -| Worker-owned reusable builds | `testing-policy.md:210` `^worker-build-lifetime` | See refinement 4. | -| Built application by default for `ProductServer` acceptances, scoped | `testing-policy.md:212` `^product-acceptance-built-app` | Excludes `appURLForServer()`, fixtures and Electron seams, as converged. | - -Other checks: the nine new anchors are unique in the tree; every new link resolves to an existing heading (`Resource diagnostics`, `Abandoned-run recovery`, `Locking`, `Temporary files`, `Destructive fixture placement`, `Pool`); the index's line counts match the files; `preflight.md:47`, `test-runner.md:153` and `sharded-execution.md:321` still describe the stale-owner exception correctly; the test-runner proof's "retains the lease until TTL" (`proofs/arch/test-runner/blaxel-testshards.md:57`) is the expected proof-stage follow-up, which the response defers on purpose. - -## Refinements (none blocking) - -1. **One word for the thing the coordinator holds.** The Blaxel spec now says *reservation* 17 times, *lease* 14 and *lock* 14 for the same object, and `test-runner.md:153` and `sharded-execution.md` still say *lease*. A cold reader will look for a distinction that does not exist. Keep *lease* (the established term) for what the coordinator holds and *lock* for the directory that records it, and drop *reservation*. The same applies to *attempt*: `terms.md:124` defines it as one execution of the shard set, while the new text uses it for one shard's execution (`attempt directory`, `attempt owner token`, `incoming attempt`). Say *shard attempt* where that is meant, as `reporting.md:218` already does. - -2. **An unreadable lock record is treated two ways.** `blaxel-testshards.md:57` lets ordinary acquisition reclaim an acquisition interrupted before registration after a clean worker audit, since execution cannot precede registration. `:63` says that for deletion "an unreadable ownership record is not proof of expiry", so the pool script would refuse a worker a test run may take. One rule serves both: a record that cannot be read is reclaimed, by acquisition or deletion, only after a clean worker audit, and held otherwise. Say it once, in the recovery section, and let the deletion paragraph refer to it. - -3. **Deletion of a worker that cannot run commands is undefined.** Deletion "checks the same identity and deadline under the guard", which needs a worker that answers `process.exec`. A sandbox that is not `DEPLOYED` cannot be leased (`:25`), so nothing can be running on it and it can be deleted without a guard; a `DEPLOYED` worker whose commands fail is the provider failure the spec sends to an operator, and deletion is that operator's tool. One sentence closes this: a worker no coordinator can lease may be deleted without a reservation. - -4. **`^worker-build-lifetime` says more than the rule.** The need is one sentence: a reusable build a helper creates belongs to the worker that created it, is published only after it succeeds, and is removed at worker teardown rather than per test case. The rest, waiting for pending builds, stopping consumers, Playwright and Vitest both, the per-launch copy and home, idempotence, is what a competent implementer does without being told, and each clause is a line every future helper change is reviewed against. Trim to the rule and the backstop link. - -5. **Consequences for the unchanged operations text.** `:158` still says to provision `poc` by deleting the complete pool, which under `:63` now refuses any worker with a live lease. Say that recreation needs an idle pool, or that refused workers are reported and the command exits non-zero. Relatedly, the `Output` section (`:141`) lists what a shard download adds but not where resource sidecars and recovery receipts land; one clause (under `artifacts/shard-/`, with the task artifacts, as `sharded-execution.md:228` implies) saves an implementer a guess, and the shard lifecycle's step list (`:86`) should name the cleanup step if its counts and free-space figures are recorded as a setup timing, as `:76` requires. - -6. **Mark the legacy paragraph transitional and drop its non-requirements.** `:61` is a mixed-version rule that ends when every branch has integrated this change. Saying so lets a later change delete it with confidence. Its last sentence, "No pool-wide drain or simultaneous branch update is required", and `:63`'s "Read-only pool listing needs no reservation" state what the spec does not require, which spec policy leaves to silence. - -7. **Two items in the Blaxel `Testing` section fail the testing-guidance test.** "Existing `isolated-github-only` fixtures retain their placement restriction" restates a rule that applies regardless (`test-runner.md:191`); removing it changes no derivation. Recording the launched browser version and executable is investigation hygiene for interpreting measurements, not something a proof needs to prove `^blaxel-scratch`; keep the `TMPDIR` observation, which a test author would miss, and move the browser-version note to the proposal's validation plan where it already lives. - -8. **Small wording.** `:69` says setup sidecars "use the attempt directory" and `:78` removes that directory before download; the two agree only through "finalize evidence outside scratch" between them. Say that setup sidecars are written in the attempt directory and finalized into the results tree before it is removed. `:76`'s last sentence ("Do not equate allocated bytes removed with free-space change…") is a caution about reading the numbers, not a requirement; the proposal can keep it. - -## Reviewer checklist - -- **Slop-free / authority.** The deltas stay in the owners the proposal named; the Blaxel spec's opening line and paragraph now claim recovery and temporary files, and reporting's opening claims resource diagnostics. The TTL escape and shell-exit unlock are replaced, not left beside the new rule. -- **Altitude and contract vs. implementation.** The TypeScript fields are the only type change and match on both sides. The Blaxel spec is where the lifecycle detail belongs; the testing policy is one place where the detail outruns the rule (refinement 4). -- **Writing and terminology.** Refinement 1. Otherwise plain, with rationale kept short. -- **Testing guidance.** Refinement 7. The new testing-policy default is a default, not a prohibition, and is honored by test design rather than asserted by tests. -- **References.** All new links and anchors resolve; nothing links to the removed sentences. -- **Size.** The Blaxel spec grows from 131 to 174 lines, in proportion to what it now decides; it is not a decomposition candidate. -- **Status.** The specs promise behavior the coordinator does not yet have, which is the workflow's normal state between spec and implementation. The one transitional rule is in refinement 6. -- **Proof coverage and mocking.** Not at this stage; the response is right that proofs follow spec convergence. - -## Evidence and limits - -- Read the full spec diff and the full Blaxel spec at `cd8aa453`; the proposal and response diffs since `57a701d0`; the merge commit and its content relative to both parents; spec policy's testing-guidance and transitional-status rules; `terms.md`; the preflight, test-runner, sharded-execution and reporting passages that cite leases or stale owners; the test-runner proof line that cites TTL; and the SDK's sandbox update methods. -- Not established: whether Blaxel accepts a label change on a `DEPLOYED` sandbox in practice (the SDK exposes the call); whether recovery fits inside the coordinator's current 30-second acquisition command timeout (an implementation choice the spec does not constrain). - -## Housekeeping - -- Read-only: no edits, commits or pushes; nothing run on this host or on Blaxel. `git fetch origin` updated remote-tracking refs. -- Primitive produced no output during this review, so I have nothing to report on its usefulness. The response records Primitive output in the author's session; I have not seen it and do not assess it. From 6b5721cb1bb6ea97737d40290b54923e9d43907b Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:07:58 +0000 Subject: [PATCH 58/64] test: reproduce unprivileged worker lease audits --- test/repo/blaxel-worker-lease.test.ts | 49 ++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/test/repo/blaxel-worker-lease.test.ts b/test/repo/blaxel-worker-lease.test.ts index 27a04e34..5eaf91a2 100644 --- a/test/repo/blaxel-worker-lease.test.ts +++ b/test/repo/blaxel-worker-lease.test.ts @@ -1,5 +1,5 @@ import { spawn, spawnSync, execFile } from "node:child_process"; -import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync, existsSync } from "node:fs"; +import { chownSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync, existsSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; @@ -10,11 +10,15 @@ import { workerLeaseCommand, workerEntryCommand, shardScratchPath } from "../../ // fixture-token filter substitutes only the whole-worker audit's scope. const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); -function fixture() { - const tmpRoot = mkdtempSync(path.join(os.tmpdir(), "lease-seam-")); roots.push(tmpRoot); +function fixture(unprivileged = false) { + // A root Blaxel worker must reproduce the unprivileged CI runner too. Use + // /tmp directly because the outer shard's private TMPDIR is not traversable. + const credentials = unprivileged && process.getuid?.() === 0 ? { uid: 65534, gid: 65534 } : {}; + const tmpRoot = mkdtempSync(path.join(unprivileged ? "/tmp" : os.tmpdir(), "lease-seam-")); roots.push(tmpRoot); + if (credentials.uid !== undefined) chownSync(tmpRoot, credentials.uid, credentials.gid!); const input = { lockId: "fixture-lock", runId: "fixture-run", shardIndex: 1, shardTotal: 1, attemptId: "fixture-attempt", owner: "fixture", expiresAt: Math.floor(Date.now() / 1000) + 1200, tmpRoot, resultsRoot: path.join(tmpRoot, "results"), auditTokens: ["fixture-lock", "unrelated-lock", "new-lock"] }; const recordPath = path.join(tmpRoot, "tv-testshard.lock", "owner.json"); - const run = (extra = {}) => spawnSync("bash", ["-c", workerLeaseCommand({ ...input, ...extra })], { encoding: "utf8", env: process.env, timeout: 15_000 }); + const run = (extra = {}) => spawnSync("bash", ["-c", workerLeaseCommand({ ...input, ...extra })], { encoding: "utf8", env: process.env, timeout: 15_000, ...credentials }); return { input, recordPath, run, record: () => JSON.parse(readFileSync(recordPath, "utf8")) }; } @@ -28,6 +32,43 @@ test.skipIf(process.platform !== "linux")("acquisition writes a renewable deadli expect(f.record().recoverAfter - f.record().lastRenewal).toBe(1_200_000); }); +test.skipIf(process.platform !== "linux")("fixture audit runs without root access to other users' environments", () => { + const f = fixture(true); + const acquired = f.run(); + expect(acquired.status, acquired.stderr).toBe(0); + expect(f.record().lockId).toBe(f.input.lockId); +}); + +test.skipIf(process.platform !== "linux")("an unreadable whole-worker audit preserves the lease and its evidence", () => { + const f = fixture(true); + expect(f.run().status).toBe(0); + const before = readFileSync(f.recordPath, "utf8"); + const sentinel = path.join(f.input.tmpRoot, "television-web-bundle-preserve"); + mkdirSync(sentinel); + // No fixture scope: production must refuse reuse when ownership is unknown. + const prepared = f.run({ operation: "prepare", auditTokens: undefined }); + expect(prepared.status).toBe(2); + expect(prepared.stderr).toMatch(/worker owner audit unreadable at PID \d+ \(EACCES\); worker reuse refused/); + expect(readFileSync(f.recordPath, "utf8")).toBe(before); + expect(existsSync(sentinel)).toBe(true); +}); + +test.skipIf(process.platform !== "linux" || process.getuid?.() !== 0)("fixture scope still refuses an unreadable process with its own user ID", async () => { + const f = fixture(true); + const ready = path.join(f.input.tmpRoot, "unreadable-ready"); + // Changing credentials after exec disables dumpability. procfs files become + // root-owned even though /proc/PID/status still identifies UID 65534. + const source = `process.setgid(65534);process.setuid(65534);require('fs').writeFileSync(${JSON.stringify(ready)},'ready');setInterval(()=>{},1000)`; + const child = spawn(process.execPath, ["-e", source], { stdio: "ignore", env: { ...process.env, TV_TEST_SHARD_OWNER: "fixture-lock" } }); + try { + await until(() => existsSync(ready)); + const acquired = f.run(); + expect(acquired.status).toBe(2); + expect(acquired.stderr).toContain(`worker owner audit unreadable at PID ${child.pid} (EACCES); worker reuse refused`); + expect(existsSync(f.recordPath)).toBe(false); + } finally { child.kill("SIGKILL"); await until(() => child.signalCode !== null); } +}); + test.skipIf(process.platform !== "linux")("a quiet expired renewable lease recovers through cleanup before replacement", () => { const f = fixture(); mkdirSync(path.dirname(f.recordPath)); From 704f484fc16d2597eeeb7b9331ba5bf2924d89f4 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:10:47 +0000 Subject: [PATCH 59/64] fix: scope fixture process audits before reading environments --- docs/working/harness-ci-validation.md | 15 +++++++++++++++ proofs/arch/test-runner/blaxel-testshards.md | 4 +++- scripts/test/blaxel-worker-lease.mjs | 14 +++++++++++++- test/repo/blaxel-worker-lease.test.ts | 2 +- 4 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 docs/working/harness-ci-validation.md diff --git a/docs/working/harness-ci-validation.md b/docs/working/harness-ci-validation.md new file mode 100644 index 00000000..e4e489b0 --- /dev/null +++ b/docs/working/harness-ci-validation.md @@ -0,0 +1,15 @@ +# Worker audit permission regression + +The ready-PR unit job at `ca2f9024` failed all seven Linux worker-lease seam cases. It ran as an unprivileged GitHub user; each acquisition attempted to read `/proc/1/environ` before its fixture-token filter. The production helper's refusal on unknown ownership follows the converged release/recovery specification. Ignoring all permission errors would instead permit reuse without a clean audit. + +The fixture must scope observations to its own operating-system user before reading process environments, as well as its declared owner tokens afterwards. Production retains a whole-worker audit with explicit refusal on unreadable ownership. Credential-changing processes matter: Linux may expose root-owned procfs files for a nondumpable process whose actual UID is still the fixture's user. Read the UIDs from procfs status, not directory ownership. + +Primary Linux documentation: [environ access check](https://man7.org/linux/man-pages/man5/proc_pid_environ.5.html), [procfs ownership and dumpability](https://man7.org/linux/man-pages/man5/proc_pid.5.html). + +## Validation + +- Red revision: `6b5721cb`. Runs only on private Blaxel pool `rose-cleanup` from dedicated coordinator `tv-cleanup-coordinator-rose`. +- Red run `2026-10-27T22-09-09-927Z-p2368-re5fbca05226aef55`: one selected failure, exactly `EACCES: permission denied, open '/proc/1/environ'`. All 22 artifacts (106,383 bytes) downloaded with size checks. Command: `npm test -- blaxel --pool rose-cleanup --file test/repo/blaxel-worker-lease.test.ts --grep "fixture audit runs without root access" --run-dir-output /tmp/rose-ci-red-run.txt`. +- Regression runs the real generated command under UID 65534 when the test host is root. Additional checks require an unreadable whole-worker audit to retain the lease and leftovers, and a same-user nondumpable child to remain an audit failure. +- No tests, builds, static checks, servers or browsers run on the shared development host. +- Primitive pre-commit discovery listed existing recovery/ownership decisions and reported truncation. This is incomplete discovery, not an authority clearance. The correction follows the converged fail-closed spec and changes no standing policy. diff --git a/proofs/arch/test-runner/blaxel-testshards.md b/proofs/arch/test-runner/blaxel-testshards.md index 9355135e..1be73b28 100644 --- a/proofs/arch/test-runner/blaxel-testshards.md +++ b/proofs/arch/test-runner/blaxel-testshards.md @@ -31,7 +31,7 @@ Pool discovery also has a contract test that runs the shared discovery helper us The SDK reads `BL_API_URL`, `BL_API_KEY`, and `BL_WORKSPACE`; the discovery contract test supplies a loopback API and fixture credentials in an isolated working directory and home, with an explicit child environment that preserves the test owner token and disables SDK telemetry and HTTP/2. Production uses the configured Blaxel API and credentials. -The worker lock and temporary-cleanup helpers accept filesystem roots for disposable seam fixtures; production uses the specified worker paths. Their decision layer accepts a clock, owner-audit/signal operations, and filesystem probes for contract failure injection; production uses worker time, procfs, signals, and the filesystem. +The worker lock and temporary-cleanup helpers accept filesystem roots for disposable seam fixtures; production uses the specified worker paths. The lease fixture's `auditTokens` scope first limits process inspection to its effective user's real, effective, saved or filesystem UIDs from procfs status, then filters readable environments to the supplied tokens. Production supplies no token filter and audits all users. The scoped fixture forfeits whole-worker coverage to the live acceptance. Their decision layer accepts a clock, owner-audit/signal operations, and filesystem probes for contract failure injection; production uses worker time, procfs, signals, and the filesystem. A coordinator milestone hook, enabled only by `TV_TEST_RUNNER_SELFTEST`, can wait on a fixture-controlled rendezvous immediately before report download while renewal continues; ordinary runs do not wait. It changes scheduling, not lease state, process results, or report bytes. Live recovery uses the existing `--lock-ttl-seconds` override to shorten the abandonment wait, not a fake clock or altered retry budget. @@ -84,6 +84,8 @@ The remaining coverage lives in `test/repo/test-runner-provider.test.ts`. - **Contract — cleanup uncertainty.** The cleaner's decision layer consumes authored filesystem-probe results and removal errors. An unexpected mount boundary is skipped, and an incomplete removal is reported without claiming the whole directory was reclaimed; neither permits new test dispatch under [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#Temporary files). These mocked conditions forfeit actual filesystem operations to the cleanup seam and live retrieval assertion; a real mount or full disk adds no necessary coverage. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “mount boundaries and incomplete removal remain reported failures without reclaimed counts” and “cleanup failures keep the old identity, while explicit deletion reserves an expired worker”) ^blaxel-t-scratch-errors +- **Seam — unreadable process ownership.** The generated lock command runs without root privileges against real Linux procfs. On a root test worker the fixture drops the command to UID/GID 65534; otherwise it uses the current user. Its user/token-scoped audit can acquire without reading other users' environments. The unscoped production audit refuses preparation when a process environment cannot be read, preserves the lease and known leftovers, and reports the PID and permission failure without environment contents. A root-only case changes a child's credentials after exec to make that same-user process nondumpable: the scoped audit must still refuse it, even though its procfs files are root-owned. This checks [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release)'s unreadable-audit rule. Credential changes and procfs reads are real; the root-only case is skipped on unprivileged CI and runs on the private Blaxel worker. (policy-grade tests: `test/repo/blaxel-worker-lease.test.ts` “fixture audit runs without root access to other users' environments”, “an unreadable whole-worker audit preserves the lease and its evidence”, and “fixture scope still refuses an unreadable process with its own user ID”) ^blaxel-t-unreadable-audit + Shared live declaration: the following acceptances run in `experiment:blaxel-harness` via `npm test -- local --surface experiment:blaxel-harness` on a dedicated credentialed Blaxel coordinator sandbox outside the worker pool, from a committed and pushed checkout, never a shared developer host or a leased worker, under [harness placement](../../../specs/arch/test-runner/test-runner.md#^blaxel-harness-placement). The caller supplies `TV_BLAXEL_HARNESS_COORDINATOR` and `TV_BLAXEL_HARNESS_POOL` as described in the [harness instructions](../../../test/runner-fixtures/blaxel-harness/README.md); no personal resource name is prescribed. They spawn the canonical CLI with real argv and inspect its exit and retained run directory. A fresh dedicated pool, at least two committed native fixture files in `experiment:blaxel-harness-fixtures`, known leftover directories and unrelated sentinels are fixtures. The two files permit the planned run's file-atomic two-shard assignment. Under [surface selection](../../../specs/arch/test-runner/test-registry.md#^blaxel-harness-surfaces), `npm test -- blaxel --tag blaxel-harness-fixture --pool --shards 2 --allow-shard-count-override 1` delegates `--suite selection --surfaces experiment:blaxel-harness-fixtures`; `npm test -- blaxel --surface experiment:blaxel-harness-fixtures --pool ` takes the targeted path. Workers read the committed registry. The coordinator, SDK, worker scripts, native runners, transport, ownership audits and filesystem operations are real. The report-download rendezvous above is the only scheduling hook; no mechanism is mocked. A bounded test driver owns and finally cleans up its coordinators and private workers. These tests do not invoke fixtures restricted to isolated GitHub hosts. - **Acceptance — reports before reuse.** A small planned run with at least two shards traverses real acquisition, architecture verification, checkout, runtime/dependency setup, execution and log/report transport under [the shard lifecycle](../../../specs/arch/test-runner/blaxel-testshards.md#Shard lifecycle). Each uploaded script path lies inside its current shard attempt directory. The retained setup phases include `temporary-cleanup`, with cleanup counts and free-space evidence. While it waits at the download rendezvous, observed renewals keep the leases valid; a competing canonical run cannot acquire the occupied capacity or alter its result files. After retrieval, the original native reports, setup/cleanup evidence and diagnostics have the original identities and readable local paths, scratch is gone, and the workers become available to a targeted run. Legacy-prefix and stale-scratch fixtures disappear, unrelated sentinels and persistent caches survive, and the next run does not accumulate an older results tree. The targeted run crosses its distinct dispatch path too. This proves [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release) and [scratch lifetime](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-scratch); the shared diagnostics observations are owned by [reporting](reporting.md#^reporting-t-resource-live). (policy-grade test: `test/runner-fixtures/blaxel-harness/driver/harness.test.ts` “qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle”, with `fixtures/outcomes.test.ts` and `fixtures/temporary.test.ts` under the same harness directory) ^blaxel-t-retrieval diff --git a/scripts/test/blaxel-worker-lease.mjs b/scripts/test/blaxel-worker-lease.mjs index 1b2bcd3c..3e18fdb5 100644 --- a/scripts/test/blaxel-worker-lease.mjs +++ b/scripts/test/blaxel-worker-lease.mjs @@ -56,6 +56,15 @@ export async function workerLeaseOperation(input, hooks = {}) { // or environments enter the evidence. Kernel threads and zombies own no work. function readProcess(pid) { const base = `/proc/${pid}`; + if (input.auditTokens) { + // Fixture tokens scope only processes belonging to the fixture user. + // Read actual UIDs: nondumpable processes have root-owned procfs files. + // Production has no token filter and must audit every user's processes. + const status = fs.readFileSync(`${base}/status`, "utf8"); + const uids = status.match(/^Uid:\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)$/m); + if (!uids) throw new Error("unreadable process user identity"); + if (!uids.slice(1).some((uid) => Number(uid) === process.geteuid())) return null; + } const stat = fs.readFileSync(`${base}/stat`, "utf8"); const fields = stat.slice(stat.lastIndexOf(")") + 1).trim().split(/\s+/); if (fields.length < 20) throw new Error("unreadable process identity"); @@ -75,7 +84,10 @@ export async function workerLeaseOperation(input, hooks = {}) { for (const name of fs.readdirSync("/proc")) { if (!/^\d+$/.test(name)) continue; try { const entry = readProcess(Number(name)); if (entry) owners.push(entry); } - catch (error) { if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; } + catch (error) { + if (["EACCES", "EPERM"].includes(error.code)) throw new Error(`worker owner audit unreadable at PID ${name} (${error.code}); worker reuse refused`); + if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; + } } return owners; } diff --git a/test/repo/blaxel-worker-lease.test.ts b/test/repo/blaxel-worker-lease.test.ts index 5eaf91a2..d4dbf768 100644 --- a/test/repo/blaxel-worker-lease.test.ts +++ b/test/repo/blaxel-worker-lease.test.ts @@ -7,7 +7,7 @@ import { afterEach, expect, test } from "vitest"; import { workerLeaseCommand, workerEntryCommand, shardScratchPath } from "../../scripts/test/blaxel-worker-lease.mjs"; // Real generated lock command, Linux flock/procfs and disposable files. The -// fixture-token filter substitutes only the whole-worker audit's scope. +// fixture-user/token filter substitutes only the whole-worker audit's scope. const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); function fixture(unprivileged = false) { From 34ee305bd01324f91fa1bfbaaadecc53cdde4745 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:13:53 +0000 Subject: [PATCH 60/64] Record Blaxel permission regression validation --- docs/working/harness-ci-validation.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/working/harness-ci-validation.md b/docs/working/harness-ci-validation.md index e4e489b0..3b536c62 100644 --- a/docs/working/harness-ci-validation.md +++ b/docs/working/harness-ci-validation.md @@ -11,5 +11,7 @@ Primary Linux documentation: [environ access check](https://man7.org/linux/man-p - Red revision: `6b5721cb`. Runs only on private Blaxel pool `rose-cleanup` from dedicated coordinator `tv-cleanup-coordinator-rose`. - Red run `2026-10-27T22-09-09-927Z-p2368-re5fbca05226aef55`: one selected failure, exactly `EACCES: permission denied, open '/proc/1/environ'`. All 22 artifacts (106,383 bytes) downloaded with size checks. Command: `npm test -- blaxel --pool rose-cleanup --file test/repo/blaxel-worker-lease.test.ts --grep "fixture audit runs without root access" --run-dir-output /tmp/rose-ci-red-run.txt`. - Regression runs the real generated command under UID 65534 when the test host is root. Additional checks require an unreadable whole-worker audit to retain the lease and leftovers, and a same-user nondumpable child to remain an audit failure. +- Green revision `704f484f`: whole `blaxel-worker-lease.test.ts` passed 10/10 in run `2026-10-27T22-11-06-608Z-p2970-r1fab74ce978afb24`; adjacent `blaxel-lease-contracts.test.ts` passed 10/10 in run `2026-10-27T22-11-39-709Z-p3087-r4ec4d055bae07cf6`. Both used the canonical Blaxel runner with default retry settings on the private pool, first infrastructure attempt, no test failures/recoveries, leaks or infrastructure failures. Downloaded and size-checked all 44 artifacts (165,216 bytes). +- One closing renewal warning occurred in the 31-second worker-lease run. Read-only SDK history shows release PID 6099 succeeded (teardown confirmed), then renewal PID 6510 failed with exit 42, both finishing at 22:11:38 UTC. The contract run's release PID 9983 succeeded at 22:11:49. Source inspection confirms renewal stops in the finalizer after release, allowing this diagnostic race. Results were already retained and worker reuse was safe; this warning does not establish loss during active ownership. The history is retained in `/tmp/rose-ci-worker-history.json`. - No tests, builds, static checks, servers or browsers run on the shared development host. - Primitive pre-commit discovery listed existing recovery/ownership decisions and reported truncation. This is incomplete discovery, not an authority clearance. The correction follows the converged fail-closed spec and changes no standing policy. From 4a285979f8a2f542904def82cb866b24dbf8878b Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:14:06 +0000 Subject: [PATCH 61/64] Prepare permission regression fix for CI Remove the completed validation checkpoint under pre-merge docs prep. The private Blaxel worker-lease and lease-contract files passed all 20 tests at 704f484f after reproducing the permission failure at 6b5721cb. The proof retains the scope and limits of the added process-permission cases. --- docs/working/harness-ci-validation.md | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 docs/working/harness-ci-validation.md diff --git a/docs/working/harness-ci-validation.md b/docs/working/harness-ci-validation.md deleted file mode 100644 index 3b536c62..00000000 --- a/docs/working/harness-ci-validation.md +++ /dev/null @@ -1,17 +0,0 @@ -# Worker audit permission regression - -The ready-PR unit job at `ca2f9024` failed all seven Linux worker-lease seam cases. It ran as an unprivileged GitHub user; each acquisition attempted to read `/proc/1/environ` before its fixture-token filter. The production helper's refusal on unknown ownership follows the converged release/recovery specification. Ignoring all permission errors would instead permit reuse without a clean audit. - -The fixture must scope observations to its own operating-system user before reading process environments, as well as its declared owner tokens afterwards. Production retains a whole-worker audit with explicit refusal on unreadable ownership. Credential-changing processes matter: Linux may expose root-owned procfs files for a nondumpable process whose actual UID is still the fixture's user. Read the UIDs from procfs status, not directory ownership. - -Primary Linux documentation: [environ access check](https://man7.org/linux/man-pages/man5/proc_pid_environ.5.html), [procfs ownership and dumpability](https://man7.org/linux/man-pages/man5/proc_pid.5.html). - -## Validation - -- Red revision: `6b5721cb`. Runs only on private Blaxel pool `rose-cleanup` from dedicated coordinator `tv-cleanup-coordinator-rose`. -- Red run `2026-10-27T22-09-09-927Z-p2368-re5fbca05226aef55`: one selected failure, exactly `EACCES: permission denied, open '/proc/1/environ'`. All 22 artifacts (106,383 bytes) downloaded with size checks. Command: `npm test -- blaxel --pool rose-cleanup --file test/repo/blaxel-worker-lease.test.ts --grep "fixture audit runs without root access" --run-dir-output /tmp/rose-ci-red-run.txt`. -- Regression runs the real generated command under UID 65534 when the test host is root. Additional checks require an unreadable whole-worker audit to retain the lease and leftovers, and a same-user nondumpable child to remain an audit failure. -- Green revision `704f484f`: whole `blaxel-worker-lease.test.ts` passed 10/10 in run `2026-10-27T22-11-06-608Z-p2970-r1fab74ce978afb24`; adjacent `blaxel-lease-contracts.test.ts` passed 10/10 in run `2026-10-27T22-11-39-709Z-p3087-r4ec4d055bae07cf6`. Both used the canonical Blaxel runner with default retry settings on the private pool, first infrastructure attempt, no test failures/recoveries, leaks or infrastructure failures. Downloaded and size-checked all 44 artifacts (165,216 bytes). -- One closing renewal warning occurred in the 31-second worker-lease run. Read-only SDK history shows release PID 6099 succeeded (teardown confirmed), then renewal PID 6510 failed with exit 42, both finishing at 22:11:38 UTC. The contract run's release PID 9983 succeeded at 22:11:49. Source inspection confirms renewal stops in the finalizer after release, allowing this diagnostic race. Results were already retained and worker reuse was safe; this warning does not establish loss during active ownership. The history is retained in `/tmp/rose-ci-worker-history.json`. -- No tests, builds, static checks, servers or browsers run on the shared development host. -- Primitive pre-commit discovery listed existing recovery/ownership decisions and reported truncation. This is incomplete discovery, not an authority clearance. The correction follows the converged fail-closed spec and changes no standing policy. From 294f2562872b4a5d41115da908ee95d99f8ccbc6 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:18:57 +0000 Subject: [PATCH 62/64] test: reproduce unreadable same-user processes outside the fixture --- test/repo/blaxel-worker-lease.test.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/test/repo/blaxel-worker-lease.test.ts b/test/repo/blaxel-worker-lease.test.ts index d4dbf768..3f424a97 100644 --- a/test/repo/blaxel-worker-lease.test.ts +++ b/test/repo/blaxel-worker-lease.test.ts @@ -32,11 +32,18 @@ test.skipIf(process.platform !== "linux")("acquisition writes a renewable deadli expect(f.record().recoverAfter - f.record().lastRenewal).toBe(1_200_000); }); -test.skipIf(process.platform !== "linux")("fixture audit runs without root access to other users' environments", () => { +test.skipIf(process.platform !== "linux")("fixture audit runs without root access to unrelated process environments", async () => { const f = fixture(true); - const acquired = f.run(); - expect(acquired.status, acquired.stderr).toBe(0); - expect(f.record().lockId).toBe(f.input.lockId); + const ready = path.join(f.input.tmpRoot, "unrelated-ready"); + // A same-user service can be nondumpable too. It belongs outside this + // fixture's process set even though a user filter would include it. + const child = process.getuid?.() === 0 ? spawn(process.execPath, ["-e", `process.setgid(65534);process.setuid(65534);require('fs').writeFileSync(${JSON.stringify(ready)},'ready');setInterval(()=>{},1000)`], { stdio: "ignore", env: process.env }) : null; + try { + if (child) await until(() => existsSync(ready)); + const acquired = f.run(); + expect(acquired.status, acquired.stderr).toBe(0); + expect(f.record().lockId).toBe(f.input.lockId); + } finally { if (child) { child.kill("SIGKILL"); await until(() => child.signalCode !== null); } } }); test.skipIf(process.platform !== "linux")("an unreadable whole-worker audit preserves the lease and its evidence", () => { From 2156f2667fc6c3112391ac753a8524badfabc629 Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:22:11 +0000 Subject: [PATCH 63/64] fix: enumerate only explicit fixture processes in lease seams --- docs/working/harness-ci-permissions.md | 11 +++++++++++ proofs/arch/test-runner/blaxel-testshards.md | 6 +++--- scripts/test/blaxel-worker-lease.d.mts | 2 +- scripts/test/blaxel-worker-lease.mjs | 20 ++++++-------------- test/repo/blaxel-worker-lease.test.ts | 11 ++++++----- 5 files changed, 27 insertions(+), 23 deletions(-) create mode 100644 docs/working/harness-ci-permissions.md diff --git a/docs/working/harness-ci-permissions.md b/docs/working/harness-ci-permissions.md new file mode 100644 index 00000000..62fdf1a1 --- /dev/null +++ b/docs/working/harness-ci-permissions.md @@ -0,0 +1,11 @@ +# Fixture process-scope correction + +CI run 40593278197 at `4a285979` exposed unreadable PID 1315 even after filtering on user IDs. Its unit artifact is retained locally under `.test-runs/2026-10-27T22-14-58-709Z-p2449-r77a22f50db287c28`. User identity does not define the fixture's processes; a same-user process can also be unreadable. The production audit correctly refuses such uncertainty. + +The Linux seam fixture should supply its actual process IDs, including both registered supervisor and descendant and the unrelated-owner sentinel in the recovery case. This substitutes process enumeration only. Procfs identity/token reads, start-time revalidation, signals, flock and filesystem operations remain real. Production enumerates the whole worker. The proof must state that the fixture forfeits whole-worker enumeration to the live qualification. + +Red revision `294f2562` creates a same-user nondumpable process outside the fixture's process set on a root Blaxel worker, then runs acquisition unprivileged. It must reproduce the remaining failure before the process-set correction. The distinct negative case includes an unreadable process in the fixture's process set and must keep refusing acquisition. + +Red run `2026-10-27T22-20-18-339Z-p2368-rf5d94786f0cb21bd` failed exactly on that outside process (PID 4962, EACCES). The canonical file/title selection used default retries. All 22 artifacts (103,107 bytes) were downloaded and size-checked before implementation. + +All runs remain on committed and pushed revisions, through the canonical Blaxel runner, on private `rose-cleanup` workers from `tv-cleanup-coordinator-rose`. No tests or builds run on the shared host. No shared worker experiments are involved. diff --git a/proofs/arch/test-runner/blaxel-testshards.md b/proofs/arch/test-runner/blaxel-testshards.md index 1be73b28..0d0d8353 100644 --- a/proofs/arch/test-runner/blaxel-testshards.md +++ b/proofs/arch/test-runner/blaxel-testshards.md @@ -31,7 +31,7 @@ Pool discovery also has a contract test that runs the shared discovery helper us The SDK reads `BL_API_URL`, `BL_API_KEY`, and `BL_WORKSPACE`; the discovery contract test supplies a loopback API and fixture credentials in an isolated working directory and home, with an explicit child environment that preserves the test owner token and disables SDK telemetry and HTTP/2. Production uses the configured Blaxel API and credentials. -The worker lock and temporary-cleanup helpers accept filesystem roots for disposable seam fixtures; production uses the specified worker paths. The lease fixture's `auditTokens` scope first limits process inspection to its effective user's real, effective, saved or filesystem UIDs from procfs status, then filters readable environments to the supplied tokens. Production supplies no token filter and audits all users. The scoped fixture forfeits whole-worker coverage to the live acceptance. Their decision layer accepts a clock, owner-audit/signal operations, and filesystem probes for contract failure injection; production uses worker time, procfs, signals, and the filesystem. +The worker lock and temporary-cleanup helpers accept filesystem roots for disposable seam fixtures; production uses the specified worker paths. The lease fixture supplies `auditPids`, its explicit process set, before procfs inspection. Production supplies no process set and enumerates the whole worker. This substitutes enumeration only and forfeits whole-worker coverage to the live acceptance; identity/token reads and signal revalidation remain real. Their decision layer accepts a clock, owner-audit/signal operations, and filesystem probes for contract failure injection; production uses worker time, procfs, signals, and the filesystem. A coordinator milestone hook, enabled only by `TV_TEST_RUNNER_SELFTEST`, can wait on a fixture-controlled rendezvous immediately before report download while renewal continues; ordinary runs do not wait. It changes scheduling, not lease state, process results, or report bytes. Live recovery uses the existing `--lock-ttl-seconds` override to shorten the abandonment wait, not a fake clock or altered retry budget. @@ -78,13 +78,13 @@ The remaining coverage lives in `test/repo/test-runner-provider.test.ts`. - **Contract — coordinator ordering and failures.** The coordinator's lease lifecycle uses recording/failing transport, download, and cleanup callbacks. Under [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release), renewal spans acquisition, execution, cleanup, and retrieval. A successful shell exit still requires descendant confirmation; unknown dispatch or failed cleanup cannot release. Successful retrieval and identity validation precede release; a download retry neither reruns tests nor resets worker results. An unused lease waits for pending writes and retains produced evidence. On interruption, the authoritative interrupted report precedes cleanup/retrieval and cannot become an ordinary completion report; cleanup still runs after a report-write error. A coordinator that loses ownership stops issuing mutations and reports infrastructure failure. These callbacks forfeit API, signals, and downloads to the live retrieval/recovery assertions; existing signal-report fixtures supply breadth without another native suite. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “retrieval retries after teardown without rerunning work; uncertainty never releases”, “renewal continues while retrieval waits and stops on ownership loss”, and “preparation evidence is downloaded before an undispatched attempt releases”; `test/repo/test-runner-provider.test.ts` interrupted-report and explicit TTL-forwarding cases; `test/repo/blaxel-worker-lease.test.ts` delayed-entry case) ^blaxel-t-lease-order -- **Seam — guarded registration and ownership.** Production worker-entry and lock helpers cross real Linux process/filesystem boundaries in a private fixture root as an ordinary repository test on Linux. It requires no Blaxel API or credentials; it runs only on Linux because it uses procfs, signals and the OS guard. Two competing acquisitions yield only one owner; killing a process holding the short guard releases the OS guard without deleting another lease. A request paused before entry registration, then resumed after ownership changes, cannot touch a checkout sentinel. Old-ID renewal/release cannot change the new record. Registered setup descendants preserve the outer token when a nested surface replaces its own token; recovery stops only the attributed PID/start identities, including a descendant that ignores TERM and needs KILL. A separate live fixture owner remains untouched and prevents reuse. The fixtures are small processes, records produced by the real helper, and rendezvous files. The supplied audit filters real procfs observations to the fixture tokens, a mock of whole-worker scope that forfeits that protection to live recovery; process inspection, identity revalidation, signals and filesystem operations remain real. This proves [registration](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-reservation) and [recovery](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery), not provider dispatch. (policy-grade tests: `test/repo/blaxel-worker-lease.test.ts` competing acquisition, delayed entry, old-identity and registered-descendant recovery cases) ^blaxel-t-worker-guard +- **Seam — guarded registration and ownership.** Production worker-entry and lock helpers cross real Linux process/filesystem boundaries in a private fixture root as an ordinary repository test on Linux. It requires no Blaxel API or credentials; it runs only on Linux because it uses procfs, signals and the OS guard. Two competing acquisitions yield only one owner; killing a process holding the short guard releases the OS guard without deleting another lease. A request paused before entry registration, then resumed after ownership changes, cannot touch a checkout sentinel. Old-ID renewal/release cannot change the new record. Registered setup descendants preserve the outer token when a nested surface replaces its own token; recovery stops only the attributed PID/start identities, including a descendant that ignores TERM and needs KILL. A separate live fixture owner remains untouched and prevents reuse. The fixtures are small processes, records produced by the real helper, and rendezvous files. The supplied process set includes the registered supervisor, its reported descendant and the unrelated-owner sentinel. This mocks whole-worker enumeration and forfeits that protection to live recovery; procfs inspection, identity revalidation, signals and filesystem operations remain real. This proves [registration](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-reservation) and [recovery](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-recovery), not provider dispatch. (policy-grade tests: `test/repo/blaxel-worker-lease.test.ts` competing acquisition, delayed entry, old-identity and registered-descendant recovery cases) ^blaxel-t-worker-guard - **Seam — bounded filesystem cleanup.** The production cleaner crosses the real filesystem in a disposable parent containing both allowed legacy prefixes, stale scratch with complete and missing metadata, the current scratch, unrelated files/directories, and symlinks to outside sentinels. With a lease and clean-audit precondition supplied as a mock, it removes exactly the permitted real directories, leaves the current directory and outside sentinels intact, rejects symlink roots, and records counts, allocated bytes and free-space observations under [known leftovers](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-known-leftovers). The audit mock forfeits real disuse confirmation to the live retrieval/recovery cases. No assertion equates allocated bytes with the filesystem's free-space increase. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “only known real directories are removed, with allocated-byte evidence and outside links preserved”; `test/repo/blaxel-worker-lease.test.ts` “attempt identity and recovery evidence survive checkout resetting the result tree”) ^blaxel-t-scratch-cleanup - **Contract — cleanup uncertainty.** The cleaner's decision layer consumes authored filesystem-probe results and removal errors. An unexpected mount boundary is skipped, and an incomplete removal is reported without claiming the whole directory was reclaimed; neither permits new test dispatch under [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#Temporary files). These mocked conditions forfeit actual filesystem operations to the cleanup seam and live retrieval assertion; a real mount or full disk adds no necessary coverage. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “mount boundaries and incomplete removal remain reported failures without reclaimed counts” and “cleanup failures keep the old identity, while explicit deletion reserves an expired worker”) ^blaxel-t-scratch-errors -- **Seam — unreadable process ownership.** The generated lock command runs without root privileges against real Linux procfs. On a root test worker the fixture drops the command to UID/GID 65534; otherwise it uses the current user. Its user/token-scoped audit can acquire without reading other users' environments. The unscoped production audit refuses preparation when a process environment cannot be read, preserves the lease and known leftovers, and reports the PID and permission failure without environment contents. A root-only case changes a child's credentials after exec to make that same-user process nondumpable: the scoped audit must still refuse it, even though its procfs files are root-owned. This checks [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release)'s unreadable-audit rule. Credential changes and procfs reads are real; the root-only case is skipped on unprivileged CI and runs on the private Blaxel worker. (policy-grade tests: `test/repo/blaxel-worker-lease.test.ts` “fixture audit runs without root access to other users' environments”, “an unreadable whole-worker audit preserves the lease and its evidence”, and “fixture scope still refuses an unreadable process with its own user ID”) ^blaxel-t-unreadable-audit +- **Seam — unreadable process ownership.** The generated lock command runs without root privileges against real Linux procfs. On a root test worker the fixture drops the command to UID/GID 65534; otherwise it uses the current user. Its explicit process set permits acquisition without reading unrelated processes' environments, including a same-user nondumpable process on a root test host. The unscoped production audit refuses preparation when a process environment cannot be read, preserves the lease and known leftovers, and reports the PID and permission failure without environment contents. A root-only case changes a child's credentials after exec to make that same-user process nondumpable: the audit must still refuse it when included in the supplied process set. This checks [release](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-release)'s unreadable-audit rule. Credential changes and procfs reads are real; the root-only case is skipped on unprivileged CI and runs on the private Blaxel worker. (policy-grade tests: `test/repo/blaxel-worker-lease.test.ts` “fixture audit runs without root access to unrelated process environments”, “an unreadable whole-worker audit preserves the lease and its evidence”, and “fixture scope still refuses an unreadable process in its process set”) ^blaxel-t-unreadable-audit Shared live declaration: the following acceptances run in `experiment:blaxel-harness` via `npm test -- local --surface experiment:blaxel-harness` on a dedicated credentialed Blaxel coordinator sandbox outside the worker pool, from a committed and pushed checkout, never a shared developer host or a leased worker, under [harness placement](../../../specs/arch/test-runner/test-runner.md#^blaxel-harness-placement). The caller supplies `TV_BLAXEL_HARNESS_COORDINATOR` and `TV_BLAXEL_HARNESS_POOL` as described in the [harness instructions](../../../test/runner-fixtures/blaxel-harness/README.md); no personal resource name is prescribed. They spawn the canonical CLI with real argv and inspect its exit and retained run directory. A fresh dedicated pool, at least two committed native fixture files in `experiment:blaxel-harness-fixtures`, known leftover directories and unrelated sentinels are fixtures. The two files permit the planned run's file-atomic two-shard assignment. Under [surface selection](../../../specs/arch/test-runner/test-registry.md#^blaxel-harness-surfaces), `npm test -- blaxel --tag blaxel-harness-fixture --pool --shards 2 --allow-shard-count-override 1` delegates `--suite selection --surfaces experiment:blaxel-harness-fixtures`; `npm test -- blaxel --surface experiment:blaxel-harness-fixtures --pool ` takes the targeted path. Workers read the committed registry. The coordinator, SDK, worker scripts, native runners, transport, ownership audits and filesystem operations are real. The report-download rendezvous above is the only scheduling hook; no mechanism is mocked. A bounded test driver owns and finally cleans up its coordinators and private workers. These tests do not invoke fixtures restricted to isolated GitHub hosts. diff --git a/scripts/test/blaxel-worker-lease.d.mts b/scripts/test/blaxel-worker-lease.d.mts index 77f71222..794478b5 100644 --- a/scripts/test/blaxel-worker-lease.d.mts +++ b/scripts/test/blaxel-worker-lease.d.mts @@ -1,5 +1,5 @@ import type { Stats } from "node:fs"; -export interface LeaseInput { lockId: string; runId?: string; shardIndex?: number; shardTotal?: number; attempt?: number; attemptId?: string; owner?: string; ttlSeconds?: number; expiresAt?: number; tmpRoot?: string; resultsRoot?: string; scratch?: string; auditTokens?: string[]; audit?: boolean; operation?: string; processName?: string; pid?: number; pidFromEnv?: boolean; retrieved?: boolean; worker?: string; summaryIdentity?: Record } +export interface LeaseInput { lockId: string; runId?: string; shardIndex?: number; shardTotal?: number; attempt?: number; attemptId?: string; owner?: string; ttlSeconds?: number; expiresAt?: number; tmpRoot?: string; resultsRoot?: string; scratch?: string; auditPids?: number[]; audit?: boolean; operation?: string; processName?: string; pid?: number; pidFromEnv?: boolean; retrieved?: boolean; worker?: string; summaryIdentity?: Record } export interface CleanupReport { ok: boolean; before: {availableBytes: number; freeBytes: number}; after: {availableBytes: number; freeBytes: number}; removed: Record; skips: unknown[]; errors: unknown[] } export interface LeaseRecord { lockId: string; runId: string; attemptId: string; recoverAfter: number; lastRenewal: number; expiresAt: number; recovery: {discardedReports: boolean} | null } export function shardScratchPath(input: LeaseInput): string; diff --git a/scripts/test/blaxel-worker-lease.mjs b/scripts/test/blaxel-worker-lease.mjs index 3e18fdb5..209ff0d5 100644 --- a/scripts/test/blaxel-worker-lease.mjs +++ b/scripts/test/blaxel-worker-lease.mjs @@ -56,15 +56,6 @@ export async function workerLeaseOperation(input, hooks = {}) { // or environments enter the evidence. Kernel threads and zombies own no work. function readProcess(pid) { const base = `/proc/${pid}`; - if (input.auditTokens) { - // Fixture tokens scope only processes belonging to the fixture user. - // Read actual UIDs: nondumpable processes have root-owned procfs files. - // Production has no token filter and must audit every user's processes. - const status = fs.readFileSync(`${base}/status`, "utf8"); - const uids = status.match(/^Uid:\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)$/m); - if (!uids) throw new Error("unreadable process user identity"); - if (!uids.slice(1).some((uid) => Number(uid) === process.geteuid())) return null; - } const stat = fs.readFileSync(`${base}/stat`, "utf8"); const fields = stat.slice(stat.lastIndexOf(")") + 1).trim().split(/\s+/); if (fields.length < 20) throw new Error("unreadable process identity"); @@ -74,18 +65,19 @@ export async function workerLeaseOperation(input, hooks = {}) { const shardToken = value("TV_TEST_SHARD_OWNER"); const surfaceToken = value("TV_TEST_SURFACE_OWNER"); if (!shardToken && !surfaceToken) return null; - if (input.auditTokens && !input.auditTokens.includes(shardToken) && !input.auditTokens.includes(surfaceToken)) return null; let command = "unavailable"; try { command = path.basename(fs.readlinkSync(`${base}/exe`)); } catch (error) { if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; } return { pid, startTime: fields[19], shardToken, surfaceToken, command }; } async function scan() { const owners = []; - for (const name of fs.readdirSync("/proc")) { - if (!/^\d+$/.test(name)) continue; - try { const entry = readProcess(Number(name)); if (entry) owners.push(entry); } + // Seam fixtures supply their process set before inspection. Production + // enumerates the whole worker and refuses any unreadable ownership. + const pids = input.auditPids ?? fs.readdirSync("/proc").filter((name) => /^\d+$/.test(name)).map(Number); + for (const pid of pids) { + try { const entry = readProcess(pid); if (entry) owners.push(entry); } catch (error) { - if (["EACCES", "EPERM"].includes(error.code)) throw new Error(`worker owner audit unreadable at PID ${name} (${error.code}); worker reuse refused`); + if (["EACCES", "EPERM"].includes(error.code)) throw new Error(`worker owner audit unreadable at PID ${pid} (${error.code}); worker reuse refused`); if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; } } diff --git a/test/repo/blaxel-worker-lease.test.ts b/test/repo/blaxel-worker-lease.test.ts index 3f424a97..b2c93a05 100644 --- a/test/repo/blaxel-worker-lease.test.ts +++ b/test/repo/blaxel-worker-lease.test.ts @@ -7,7 +7,7 @@ import { afterEach, expect, test } from "vitest"; import { workerLeaseCommand, workerEntryCommand, shardScratchPath } from "../../scripts/test/blaxel-worker-lease.mjs"; // Real generated lock command, Linux flock/procfs and disposable files. The -// fixture-user/token filter substitutes only the whole-worker audit's scope. +// supplied process IDs substitute only the whole-worker audit's enumeration. const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); function fixture(unprivileged = false) { @@ -16,7 +16,7 @@ function fixture(unprivileged = false) { const credentials = unprivileged && process.getuid?.() === 0 ? { uid: 65534, gid: 65534 } : {}; const tmpRoot = mkdtempSync(path.join(unprivileged ? "/tmp" : os.tmpdir(), "lease-seam-")); roots.push(tmpRoot); if (credentials.uid !== undefined) chownSync(tmpRoot, credentials.uid, credentials.gid!); - const input = { lockId: "fixture-lock", runId: "fixture-run", shardIndex: 1, shardTotal: 1, attemptId: "fixture-attempt", owner: "fixture", expiresAt: Math.floor(Date.now() / 1000) + 1200, tmpRoot, resultsRoot: path.join(tmpRoot, "results"), auditTokens: ["fixture-lock", "unrelated-lock", "new-lock"] }; + const input = { lockId: "fixture-lock", runId: "fixture-run", shardIndex: 1, shardTotal: 1, attemptId: "fixture-attempt", owner: "fixture", expiresAt: Math.floor(Date.now() / 1000) + 1200, tmpRoot, resultsRoot: path.join(tmpRoot, "results"), auditPids: [] as number[] }; const recordPath = path.join(tmpRoot, "tv-testshard.lock", "owner.json"); const run = (extra = {}) => spawnSync("bash", ["-c", workerLeaseCommand({ ...input, ...extra })], { encoding: "utf8", env: process.env, timeout: 15_000, ...credentials }); return { input, recordPath, run, record: () => JSON.parse(readFileSync(recordPath, "utf8")) }; @@ -53,14 +53,14 @@ test.skipIf(process.platform !== "linux")("an unreadable whole-worker audit pres const sentinel = path.join(f.input.tmpRoot, "television-web-bundle-preserve"); mkdirSync(sentinel); // No fixture scope: production must refuse reuse when ownership is unknown. - const prepared = f.run({ operation: "prepare", auditTokens: undefined }); + const prepared = f.run({ operation: "prepare", auditPids: undefined }); expect(prepared.status).toBe(2); expect(prepared.stderr).toMatch(/worker owner audit unreadable at PID \d+ \(EACCES\); worker reuse refused/); expect(readFileSync(f.recordPath, "utf8")).toBe(before); expect(existsSync(sentinel)).toBe(true); }); -test.skipIf(process.platform !== "linux" || process.getuid?.() !== 0)("fixture scope still refuses an unreadable process with its own user ID", async () => { +test.skipIf(process.platform !== "linux" || process.getuid?.() !== 0)("fixture scope still refuses an unreadable process in its process set", async () => { const f = fixture(true); const ready = path.join(f.input.tmpRoot, "unreadable-ready"); // Changing credentials after exec disables dumpability. procfs files become @@ -69,7 +69,7 @@ test.skipIf(process.platform !== "linux" || process.getuid?.() !== 0)("fixture s const child = spawn(process.execPath, ["-e", source], { stdio: "ignore", env: { ...process.env, TV_TEST_SHARD_OWNER: "fixture-lock" } }); try { await until(() => existsSync(ready)); - const acquired = f.run(); + const acquired = f.run({ auditPids: [child.pid!] }); expect(acquired.status).toBe(2); expect(acquired.stderr).toContain(`worker owner audit unreadable at PID ${child.pid} (EACCES); worker reuse refused`); expect(existsSync(f.recordPath)).toBe(false); @@ -160,6 +160,7 @@ test.skipIf(process.platform !== "linux")("recovery kills registered descendants try { await until(() => existsSync(ready)); const observed = JSON.parse(readFileSync(ready, "utf8")); + f.input.auditPids.push(child.pid!, observed.pid, unrelated.pid!); expect(observed.outer).toBe(f.input.lockId); expect(observed.surface).toMatch(/^tv1\./); expect(f.record().dispatch.pid).toBeGreaterThan(0); writeFileSync(f.recordPath, JSON.stringify({ ...f.record(), recoverAfter: 1 })); From 9787188de3323591caa8cf326966d6d2b8c264dc Mon Sep 17 00:00:00 2001 From: Josh Whiting <167151+jwhiting@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:24:09 +0000 Subject: [PATCH 64/64] Prepare scoped process fixtures for CI Blaxel red/green: 294f2562 reproduces the unreadable same-user process; 2156f266 passes all 20 worker-lease and lease-contract cases with default retries, no failures, leaks or infrastructure failures. Retained run directories are 2026-10-27T22-22-48-239Z-p2976-re22b246ab160f1ce and 2026-10-27T22-23-18-728Z-p3090-r6956a6b83ed3f866. Remove the completed checkpoint under docs prep and a trailing blank line. --- docs/working/harness-ci-permissions.md | 11 ----------- scripts/test/blaxel-lease-lifecycle.mjs | 1 - 2 files changed, 12 deletions(-) delete mode 100644 docs/working/harness-ci-permissions.md diff --git a/docs/working/harness-ci-permissions.md b/docs/working/harness-ci-permissions.md deleted file mode 100644 index 62fdf1a1..00000000 --- a/docs/working/harness-ci-permissions.md +++ /dev/null @@ -1,11 +0,0 @@ -# Fixture process-scope correction - -CI run 40593278197 at `4a285979` exposed unreadable PID 1315 even after filtering on user IDs. Its unit artifact is retained locally under `.test-runs/2026-10-27T22-14-58-709Z-p2449-r77a22f50db287c28`. User identity does not define the fixture's processes; a same-user process can also be unreadable. The production audit correctly refuses such uncertainty. - -The Linux seam fixture should supply its actual process IDs, including both registered supervisor and descendant and the unrelated-owner sentinel in the recovery case. This substitutes process enumeration only. Procfs identity/token reads, start-time revalidation, signals, flock and filesystem operations remain real. Production enumerates the whole worker. The proof must state that the fixture forfeits whole-worker enumeration to the live qualification. - -Red revision `294f2562` creates a same-user nondumpable process outside the fixture's process set on a root Blaxel worker, then runs acquisition unprivileged. It must reproduce the remaining failure before the process-set correction. The distinct negative case includes an unreadable process in the fixture's process set and must keep refusing acquisition. - -Red run `2026-10-27T22-20-18-339Z-p2368-rf5d94786f0cb21bd` failed exactly on that outside process (PID 4962, EACCES). The canonical file/title selection used default retries. All 22 artifacts (103,107 bytes) were downloaded and size-checked before implementation. - -All runs remain on committed and pushed revisions, through the canonical Blaxel runner, on private `rose-cleanup` workers from `tv-cleanup-coordinator-rose`. No tests or builds run on the shared host. No shared worker experiments are involved. diff --git a/scripts/test/blaxel-lease-lifecycle.mjs b/scripts/test/blaxel-lease-lifecycle.mjs index 46118b60..4c4980de 100644 --- a/scripts/test/blaxel-lease-lifecycle.mjs +++ b/scripts/test/blaxel-lease-lifecycle.mjs @@ -61,4 +61,3 @@ export async function retainAttemptEvidence(lease, directory) { fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, raw); } } -