From 3b69d6d54cec29bbeaa129914419c90a70fb87d7 Mon Sep 17 00:00:00 2001
From: minixalpha <xkzalpha@gmail.com>
Date: Fri, 9 Oct 2026 07:38:55 -0700
Subject: [PATCH 1/7] feat(agent): test evidence-based verification guidance

---
 src/nanopycodeagent/agent.py | 37 ++++++++++++++++++++++--------------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/src/nanopycodeagent/agent.py b/src/nanopycodeagent/agent.py
index 4748125..3cae8ca 100644
--- a/src/nanopycodeagent/agent.py
+++ b/src/nanopycodeagent/agent.py
@@ -215,20 +215,29 @@ def _append_budget_note(
     "reference solutions or hidden evaluation tests from outside the supplied "
     "materials unless the task explicitly requests them. Derive checks from "
     "the stated requirements.\n\n"
-    "Verify the requested behavior through the real entry point when feasible. "
-    "Check the content and meaning of outputs, not only command success or "
-    "file existence. For logic or numerical work, choose a few representative "
-    "and boundary cases justified by the task, using an independent reference, "
-    "invariant, or consistency check when available. Do not derive every "
-    "expectation from the implementation being tested or weaken requirements "
-    "to make a check pass. Keep verification proportional to the task and "
-    "remaining budget.\n\n"
-    "When a check fails, inspect the error and relevant output, revise the "
-    "underlying assumption or implementation, and rerun the affected check. "
-    "Keep the task's constraints intact. Once appropriate checks support the "
-    "required behavior, save the deliverables and finish. State what was "
-    "verified and any remaining limitation without claiming unobserved "
-    "success.\n\n"
+    "Before implementing, identify a few high-risk requirements and how to "
+    "check them against task-provided evidence, an independently computed "
+    "expectation, or an invariant. Keep this brief. Prefer a small case through "
+    "the real dependency and entry point over a substitute model. Choose "
+    "representative and boundary cases justified by the task. For multiple "
+    "items, preserve their identities and check every item in the chosen case; "
+    "aggregate or last-item agreement can hide errors. Check observable "
+    "behavior required by the task as well as final values, without inventing "
+    "requirements. Keep verification proportional to risk and remaining budget.\n\n"
+    "When a check fails, distinguish an implementation error, a test error, "
+    "and an ambiguous requirement before changing code or expectations. "
+    "Changing an expected value, tolerance, selection, or ordering needs "
+    "support from the requirements or an independent reference; retain "
+    "coverage of the original failing behavior. Do not make the test merely "
+    "mirror the implementation. For a critical self-authored check, when "
+    "inexpensive, confirm that it rejects a deliberately incorrect result. "
+    "Repair the cause and rerun affected checks.\n\n"
+    "Before finishing, inspect the actual check results for the final saved "
+    "deliverables. A successful command exit or file existence alone does "
+    "not establish correct behavior. Identify unresolved failures and gaps "
+    "where only a substitute was checked; do not claim those requirements "
+    "passed. Once appropriate checks support the required behavior, finish "
+    "with the observed evidence and remaining limitations.\n\n"
 ) + _TOOL_GUIDANCE
 
 def _json_value(value: object) -> JsonValue:

From a79995430b9224ea7b03e2f13bd109bb6a58c7fe Mon Sep 17 00:00:00 2001
From: minixalpha <xkzalpha@gmail.com>
Date: Fri, 9 Oct 2026 07:44:54 -0700
Subject: [PATCH 2/7] test(benchmarks): register verification quality prompt
 comparison

---
 .../verification-quality-20261009/README.md   | 104 +++++
 .../build-cython-ext.json                     |  22 ++
 .../cancel-async-tasks.json                   |  22 ++
 .../control-prompt.txt                        |   9 +
 .../experiment.json                           | 357 ++++++++++++++++++
 .../install-only.json                         |  34 ++
 .../llm-inference-batching-scheduler.json     |  22 ++
 .../operations-guidance.txt                   |   2 +
 .../operations-prompt.txt                     |  11 +
 .../operations.patch                          |  12 +
 .../torch-pipeline-parallelism.json           |  22 ++
 .../verification-prompt.txt                   |  11 +
 .../verification.patch                        |  48 +++
 benchmarks/harbor/scripts/run_comparison.py   | 150 ++++++++
 benchmarks/harbor/tests/test_comparison.py    |  55 +++
 15 files changed, 881 insertions(+)
 create mode 100644 benchmarks/harbor/configs/verification-quality-20261009/README.md
 create mode 100644 benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json
 create mode 100644 benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json
 create mode 100644 benchmarks/harbor/configs/verification-quality-20261009/control-prompt.txt
 create mode 100644 benchmarks/harbor/configs/verification-quality-20261009/experiment.json
 create mode 100644 benchmarks/harbor/configs/verification-quality-20261009/install-only.json
 create mode 100644 benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json
 create mode 100644 benchmarks/harbor/configs/verification-quality-20261009/operations-guidance.txt
 create mode 100644 benchmarks/harbor/configs/verification-quality-20261009/operations-prompt.txt
 create mode 100644 benchmarks/harbor/configs/verification-quality-20261009/operations.patch
 create mode 100644 benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json
 create mode 100644 benchmarks/harbor/configs/verification-quality-20261009/verification-prompt.txt
 create mode 100644 benchmarks/harbor/configs/verification-quality-20261009/verification.patch
 create mode 100644 benchmarks/harbor/scripts/run_comparison.py
 create mode 100644 benchmarks/harbor/tests/test_comparison.py

diff --git a/benchmarks/harbor/configs/verification-quality-20261009/README.md b/benchmarks/harbor/configs/verification-quality-20261009/README.md
new file mode 100644
index 0000000..c55af8e
--- /dev/null
+++ b/benchmarks/harbor/configs/verification-quality-20261009/README.md
@@ -0,0 +1,104 @@
+# Verification quality and long-command guidance comparison
+
+This is a predeclared exploratory comparison of two independent prompt changes
+against a fresh shared control. It is not a new pilot20 score. The frozen
+[plan](experiment.json) contains every task revision, source revision, prompt
+hash, configuration hash, and scheduled job. Results must retain failures and
+infrastructure errors without replacing trials based on their outcomes.
+
+| Arm | Source revision | Headless prompt words | Change |
+| --- | --- | ---: | --- |
+| Control | `2eb99b608dfb39464d17d109a4a8f11afda607a9` | 385 | Current main, including the workspace-reference clarification |
+| Verification | `3b69d6d` | 482 | Concrete verification planning, real-entry checks, complete item coverage, justified test corrections, and final-version evidence |
+| Operations | `ca624b0` | 480 | Explicit Bash time limit, redirected background work, completion status, and intermediate deliverables |
+
+Only `HEADLESS_SYSTEM_PROMPT` differs between arms. The interactive prompt,
+tools, runtime recovery, work budgets, adapter, images, and official tests are
+unchanged. No finish gate or new runtime verification machinery is introduced.
+The candidate strategies are bundles; this experiment cannot attribute an
+observed difference to an individual sentence.
+
+## Cohort and limits
+
+Each arm runs each task twice: **24 fresh model trials** in total, with at most
+two active trials. The queue rotates arm order and runs consecutive pairs to
+completion before starting the next pair. Repeat indices are descriptive;
+provider seeds are not fixed, and pairs do not share random model samples.
+
+| Task | Selection | Work budget |
+| --- | --- | ---: |
+| `torch-pipeline-parallelism` | Prior diagnostic failure | 720 seconds |
+| `llm-inference-batching-scheduler` | Prior avoidable tool timeout | 1,620 seconds |
+| `build-cython-ext` | Outside the previous pilot20 tuning cohort | 720 seconds |
+| `cancel-async-tasks` | Outside the previous pilot20 tuning cohort | 720 seconds |
+
+The two new tasks were selected from the pinned catalog using task names and
+public requirements. The candidate text was drafted before their instructions
+were read. Their hidden tests and solutions are not inspected until all outputs
+are fixed. The experiment designer previously inspected pipeline evaluation
+code during diagnosis; that task is explicitly diagnostic. No task-specific
+backward-order rule, hidden expected answer, function name, or model parameter
+is added to either candidate.
+
+All runs use `deepseek/deepseek-flash` at the official DeepSeek Anthropic-compatible
+endpoint, 100 replies, and 65,536 output tokens per request. Temperature and
+reasoning effort are not explicitly set. Setup has a separate 1,800-second
+limit. Whole-trial retries are disabled. Only verifier timeouts may retry, with
+at most three attempts on the same agent output and the original per-attempt
+limit. Setup failures and verifier errors remain separate from scored failures.
+
+## Execution
+
+Create the three isolated checkouts listed in the plan, at their recorded refs.
+The exact prompt files and candidate patches are retained here for reproduction.
+Use new job names and a new registration directory for any later reproduction;
+never overwrite this experiment's records. Credentials belong in the process
+environment, not in configuration files or logs. Set `ANTHROPIC_API_KEY` and
+`ANTHROPIC_BASE_URL=https://api.deepseek.com/anthropic`, and leave
+`ANTHROPIC_MODEL` unset so the adapter uses the model in the configurations.
+
+From the repository root:
+
+```bash
+python benchmarks/harbor/scripts/run_comparison.py \
+  --plan benchmarks/harbor/configs/verification-quality-20261009/experiment.json \
+  --phase preflight
+
+python benchmarks/harbor/scripts/run_comparison.py \
+  --plan benchmarks/harbor/configs/verification-quality-20261009/experiment.json \
+  --phase model
+```
+
+The orchestrator invokes only the standard command from each source root:
+`uv run --project benchmarks/harbor python -m harbor_adapter run --config ...`.
+It introduces no adapter copies or bootstrap scripts. Standard job snapshots
+record the wheel and Python sources. All arms share the verified APT cache.
+
+Before any model work, run install-only checks on all four control tasks and a
+wheel-installation smoke check for each candidate. None of these tasks has a
+registered verifier-dependency profile: their status must remain explicitly
+`not_configured`, even when agent installation succeeds. Full trials repeat
+setup in fresh containers. A failed install-only check blocks model launch.
+
+The orchestrator records its registered plan and every launch, skips only jobs
+already recorded as finished, and refuses to restart a previously started job.
+Interrupted jobs require inspection; the orchestrator never silently retries
+them. A completed reward of zero is a valid result, not a reason to rerun.
+
+## Evaluation
+
+Report each candidate against the fresh control, separating the diagnostic and
+new-task cohorts. Preserve raw rewards, but exclude confirmed external
+task-answer or hidden-test exposure from independent-capability comparisons.
+Use only common eligible cases for paired comparisons.
+
+Review tool evidence for real-entry validation, complete coverage within chosen
+cases, independently justified expectations, unresolved failed checks, and
+unsupported final claims. Track checks that became stale after relevant edits.
+Missing evidence is unknown; a command name or zero exit code alone does not
+prove validation. Also report agent execution time, model attempts, tool
+timeouts, truncation/recovery, estimated cost, and accounting completeness.
+
+Two repeats and two tasks outside prior tuning are insufficient to establish a
+general capability gain. Report every regression and cost increase. Do not
+promote a candidate based solely on pipeline or one favorable trial.
diff --git a/benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json b/benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json
new file mode 100644
index 0000000..222fc73
--- /dev/null
+++ b/benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json
@@ -0,0 +1,22 @@
+{
+  "n_concurrent_trials": 1,
+  "n_attempts": 1,
+  "agents": [
+    {
+      "name": "harbor_adapter:NanoPyCodeAgent",
+      "model_name": "deepseek/deepseek-flash",
+      "override_setup_timeout_sec": 1800,
+      "kwargs": {
+        "max_turns": 100,
+        "max_tokens": 65536,
+        "time_budget_seconds": 720
+      }
+    }
+  ],
+  "tasks": [
+    {
+      "name": "terminal-bench/build-cython-ext",
+      "ref": "sha256:35dc561790e73f13e59837828ac0863c0893edc6b71d42cf7d58656c843adf12"
+    }
+  ]
+}
diff --git a/benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json b/benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json
new file mode 100644
index 0000000..7a885de
--- /dev/null
+++ b/benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json
@@ -0,0 +1,22 @@
+{
+  "n_concurrent_trials": 1,
+  "n_attempts": 1,
+  "agents": [
+    {
+      "name": "harbor_adapter:NanoPyCodeAgent",
+      "model_name": "deepseek/deepseek-flash",
+      "override_setup_timeout_sec": 1800,
+      "kwargs": {
+        "max_turns": 100,
+        "max_tokens": 65536,
+        "time_budget_seconds": 720
+      }
+    }
+  ],
+  "tasks": [
+    {
+      "name": "terminal-bench/cancel-async-tasks",
+      "ref": "sha256:a3d048d351136e48070696cda8bb79660dfd74db1fea3b6da88559f0332699c1"
+    }
+  ]
+}
diff --git a/benchmarks/harbor/configs/verification-quality-20261009/control-prompt.txt b/benchmarks/harbor/configs/verification-quality-20261009/control-prompt.txt
new file mode 100644
index 0000000..56116f5
--- /dev/null
+++ b/benchmarks/harbor/configs/verification-quality-20261009/control-prompt.txt
@@ -0,0 +1,9 @@
+You are nanoPyCodeAgent, a coding agent running non-interactively on a task handed to you up front. There is no user to reply to you: never ask a clarifying question, never stop to wait for confirmation, and never present a plan for approval — decide on your own and carry it out. Work the task through to the end, then check the result with the tools instead of assuming it worked. When it is done, answer with a short summary and no further tool calls: that reply is what ends the run. Runtime budget reminders report remaining time and model replies; when either is running low, prioritize saving the required deliverables, necessary verification, and a final summary. Do not start optional work after the requirements are satisfied.
+
+Identify the required deliverables, interfaces, and constraints. Treat examples as evidence, not as the full contract unless the task says so. Inspect relevant files, dependencies, and existing checks instead of guessing. Task-supplied workspace files, including source, executables, and tests, are available references unless the task restricts their use. Consult general documentation as needed. Do not seek or use task-specific reference solutions or hidden evaluation tests from outside the supplied materials unless the task explicitly requests them. Derive checks from the stated requirements.
+
+Verify the requested behavior through the real entry point when feasible. Check the content and meaning of outputs, not only command success or file existence. For logic or numerical work, choose a few representative and boundary cases justified by the task, using an independent reference, invariant, or consistency check when available. Do not derive every expectation from the implementation being tested or weaken requirements to make a check pass. Keep verification proportional to the task and remaining budget.
+
+When a check fails, inspect the error and relevant output, revise the underlying assumption or implementation, and rerun the affected check. Keep the task's constraints intact. Once appropriate checks support the required behavior, save the deliverables and finish. State what was verified and any remaining limitation without claiming unobserved success.
+
+Prefer the read tool for viewing files, the edit tool for changing part of an existing file, and the write tool for creating files or rewriting them whole. Use the bash tool to run commands, search with grep, and complete tasks that need real command output instead of guessing.
\ No newline at end of file
diff --git a/benchmarks/harbor/configs/verification-quality-20261009/experiment.json b/benchmarks/harbor/configs/verification-quality-20261009/experiment.json
new file mode 100644
index 0000000..45446f2
--- /dev/null
+++ b/benchmarks/harbor/configs/verification-quality-20261009/experiment.json
@@ -0,0 +1,357 @@
+{
+  "schema_version": 1,
+  "status": "predeclared",
+  "registered_at": "2026-10-09T14:40:14.377201+00:00",
+  "kind": "exploratory_three_arm_prompt_comparison",
+  "baseline_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
+  "hypotheses": {
+    "verification": "Concrete evidence selection, real-entry checks, justified test corrections, and final-version evidence review improve validation quality.",
+    "operations": "Explicit Bash lifetime and completion guidance reduces avoidable tool timeouts and enables real verification."
+  },
+  "arms": {
+    "control": {
+      "source_root": "jobs/verification-quality-control-source",
+      "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
+      "prompt_file": "control-prompt.txt",
+      "prompt_sha256": "37c4d9d44640c0051dc5a20e5954a067b98e8c51be07a48f080aa059e725c55d",
+      "prompt_words": 385
+    },
+    "verification": {
+      "source_root": "jobs/verification-quality-verification-source",
+      "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
+      "prompt_file": "verification-prompt.txt",
+      "prompt_sha256": "b87d1d80fcea50083189730cc42f60baf9050ebbcb00f317cd86fb93881685e6",
+      "prompt_words": 482
+    },
+    "operations": {
+      "source_root": "jobs/verification-quality-operations-source",
+      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
+      "prompt_file": "operations-prompt.txt",
+      "prompt_sha256": "a8e9505df06f674bdc64bb2b4bd166a7917adcdcea5b517e9c707de954e3541b",
+      "prompt_words": 480
+    }
+  },
+  "model": "deepseek/deepseek-flash",
+  "endpoint": "https://api.deepseek.com/anthropic",
+  "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a",
+  "tasks": [
+    {
+      "name": "terminal-bench/torch-pipeline-parallelism",
+      "ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb",
+      "cohort": "diagnostic",
+      "work_budget_seconds": 720,
+      "verifier_timeout_seconds": 900.0,
+      "docker_image": "alexgshaw/torch-pipeline-parallelism:20251031",
+      "task_file_sha256": {
+        "README.md": "297ddea07c230dfd246a8ab07337dd7abcd8e5e2a2b2e26590cb0a8af779839c",
+        "environment/Dockerfile": "c357ea41e4a4013c5c5980b8fef04af6a5ae250ccc6697cf3bad3127401a931f",
+        "instruction.md": "989bfcf0c54baf6c8cdc391743dbb2f9e858c5779dd85112536875d4d1b2dcd2",
+        "solution/solve.sh": "787198e4178d88b887697aefdd29819f29435ffdfe356545d92f58a30fb38b70",
+        "task.toml": "8b4f9f525c08afbbc4eca9168e46238a61ccb903486773e9ca4da64bb7ffe857",
+        "tests/test.sh": "cc9fd28b6590993fc37e4a99da771cbf04726c8c62fe308e802612241da69dea",
+        "tests/test_outputs.py": "23574246a407163a8e6581c84287cc2f1eb745efbaad2a317a5a28c39958c64f"
+      },
+      "verifier_dependency_profile": "not_configured"
+    },
+    {
+      "name": "terminal-bench/llm-inference-batching-scheduler",
+      "ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67",
+      "cohort": "diagnostic",
+      "work_budget_seconds": 1620,
+      "verifier_timeout_seconds": 1800.0,
+      "docker_image": "alexgshaw/llm-inference-batching-scheduler:20251031",
+      "task_file_sha256": {
+        "README.md": "ecc9dd635357c184742cf72fc1e452a8946e207b183fc9a7eebbcf07133b911f",
+        "environment/Dockerfile": "10a6193904ee0022ac098504182989893b196dfc74e626a3ca48534a8eacde48",
+        "environment/task_file/input_data/requests_bucket_1.jsonl": "8b3cf1e2b06074162038064dd2c114abe308acd32cec7d365ee63ee81978f7de",
+        "environment/task_file/input_data/requests_bucket_2.jsonl": "e15e2d4c4d55f875c42a6042610d6282f6c0657c39d3e0d0f087268252525cb5",
+        "environment/task_file/scripts/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+        "environment/task_file/scripts/baseline_packer.py": "547d230d5e6d93197803480cb81cab0ec6ac63fcfa0a24b246f549523f216c4e",
+        "environment/task_file/scripts/cost_model.py": "0cf417fd5ff0112defcd46970d37a290c0763c5a9092e91cb3ebb86c3f20e6ce",
+        "instruction.md": "0817374767533fbf51bb93345ee1bba266263c216c6cc55c096e71b4a1d37dfc",
+        "solution/solve.sh": "cb9efb586d76a893c3027e62029e286eb6c7583dc92dbf1840a3adc236a926d7",
+        "task.toml": "52aaac942f917e6a91ff658e0272195bd1cd842d41d71e302414c52603cc588b",
+        "tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+        "tests/cost_model_for_tests.py": "d3e810a99d40327fb8317f6512d879c52c6aeebdf5dcdc0c5e117c7df61c9ab9",
+        "tests/test.sh": "005f980332355e2fe143d4ff4aac11828adf070594bbe7055399c238f06953ca",
+        "tests/test_outputs.py": "80b06b7c4ba0db28832b767d48be74bb462d7f5df5798d9cb58c4f1160d96ffa"
+      },
+      "verifier_dependency_profile": "not_configured"
+    },
+    {
+      "name": "terminal-bench/build-cython-ext",
+      "ref": "sha256:35dc561790e73f13e59837828ac0863c0893edc6b71d42cf7d58656c843adf12",
+      "cohort": "held_out_from_prior_tuning",
+      "work_budget_seconds": 720,
+      "verifier_timeout_seconds": 900.0,
+      "docker_image": "alexgshaw/build-cython-ext:20251031",
+      "task_file_sha256": {
+        "README.md": "d5f5113941c3963292bae396baa9369571026f46e59437f98ddcc93475dd7729",
+        "environment/Dockerfile": "cb6cf421cb9b69d43bd25cd491e49bda982fbb865d47515178d017eda727194e",
+        "instruction.md": "7e52874e45f88505bd21540229719de6639a401477557d814489981d472f0e34",
+        "solution/solve.sh": "41bbdde88617070366a70e518eedca466a4ea1acd7e96899b2fb7da988e40abc",
+        "task.toml": "c472f0e7522c128fb7f03e00b50c8cdf2e5e6474daa1e094e7441f4b54e770b2",
+        "tests/test.sh": "0156c9c4085258b696ce424cf1ce96cba44d1263c5d73bdf54083fd80237f915",
+        "tests/test_outputs.py": "6549f1a8e04b6163ec08877e6f6b9868bf94563f3a3c72ac1fba9c512c31689e"
+      },
+      "verifier_dependency_profile": "not_configured"
+    },
+    {
+      "name": "terminal-bench/cancel-async-tasks",
+      "ref": "sha256:a3d048d351136e48070696cda8bb79660dfd74db1fea3b6da88559f0332699c1",
+      "cohort": "held_out_from_prior_tuning",
+      "work_budget_seconds": 720,
+      "verifier_timeout_seconds": 900.0,
+      "docker_image": "alexgshaw/cancel-async-tasks:20251031",
+      "task_file_sha256": {
+        "README.md": "9adf86b1279cc44f83fe16e6d340595aa992240f1d87243d204cae32e453693b",
+        "environment/Dockerfile": "1ee7bdb4a7536f5f114c11a3ac6ff42790abfea901fcba6b2a8ea8685f19140e",
+        "instruction.md": "23d22f7039b50682941652c716f96dac314b8585c32c133614cac77dc067f47e",
+        "solution/solve.sh": "4938ca4ad5b7dcd36b414b42a1abb10c058ecefce7462b678f9bd05c9fae1844",
+        "task.toml": "14db0675e04408f2133b0067e8dbddf66a1d6e7baf6b8d38bef8ecccfc4e58a8",
+        "tests/test.py": "f533c0a64db068f582b4733c5b81f2e66b8fa52fcd3efa65752e6bd6d24780aa",
+        "tests/test.sh": "79600a6b863def31b4daa008297423010cd2d6243e689d67503e118e4dc53397",
+        "tests/test_outputs.py": "d6f510b51fafc81b4e9b5cc95535aa13491c8753e0e4171db42c45f4de460ee7"
+      },
+      "verifier_dependency_profile": "not_configured"
+    }
+  ],
+  "repeats_per_task_per_arm": 2,
+  "total_model_runs": 24,
+  "max_concurrent_trials": 2,
+  "preflight": [
+    {
+      "arm": "control",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/install-only.json",
+      "job_name": "vq-install-control-20261009",
+      "config_sha256": "d7e3397273740410996066a00c3429c8432a021c067f535d161dcb7c1d1c58b5"
+    },
+    {
+      "arm": "verification",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-install-verification-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02"
+    },
+    {
+      "arm": "operations",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-install-operations-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02"
+    }
+  ],
+  "queue": [
+    {
+      "arm": "control",
+      "repeat": 1,
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
+      "job_name": "vq-con-torch-pipeline-parallelism-r1-20261009",
+      "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4"
+    },
+    {
+      "arm": "verification",
+      "repeat": 1,
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
+      "job_name": "vq-ver-torch-pipeline-parallelism-r1-20261009",
+      "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4"
+    },
+    {
+      "arm": "operations",
+      "repeat": 1,
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
+      "job_name": "vq-ope-torch-pipeline-parallelism-r1-20261009",
+      "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4"
+    },
+    {
+      "arm": "operations",
+      "repeat": 2,
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
+      "job_name": "vq-ope-torch-pipeline-parallelism-r2-20261009",
+      "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4"
+    },
+    {
+      "arm": "verification",
+      "repeat": 2,
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
+      "job_name": "vq-ver-torch-pipeline-parallelism-r2-20261009",
+      "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4"
+    },
+    {
+      "arm": "control",
+      "repeat": 2,
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
+      "job_name": "vq-con-torch-pipeline-parallelism-r2-20261009",
+      "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4"
+    },
+    {
+      "arm": "verification",
+      "repeat": 1,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-ver-llm-inference-batching-scheduler-r1-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d"
+    },
+    {
+      "arm": "operations",
+      "repeat": 1,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-ope-llm-inference-batching-scheduler-r1-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d"
+    },
+    {
+      "arm": "control",
+      "repeat": 1,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-con-llm-inference-batching-scheduler-r1-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d"
+    },
+    {
+      "arm": "control",
+      "repeat": 2,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-con-llm-inference-batching-scheduler-r2-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d"
+    },
+    {
+      "arm": "operations",
+      "repeat": 2,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-ope-llm-inference-batching-scheduler-r2-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d"
+    },
+    {
+      "arm": "verification",
+      "repeat": 2,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-ver-llm-inference-batching-scheduler-r2-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d"
+    },
+    {
+      "arm": "control",
+      "repeat": 1,
+      "task": "terminal-bench/build-cython-ext",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
+      "job_name": "vq-con-build-cython-ext-r1-20261009",
+      "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7"
+    },
+    {
+      "arm": "verification",
+      "repeat": 1,
+      "task": "terminal-bench/build-cython-ext",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
+      "job_name": "vq-ver-build-cython-ext-r1-20261009",
+      "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7"
+    },
+    {
+      "arm": "operations",
+      "repeat": 1,
+      "task": "terminal-bench/build-cython-ext",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
+      "job_name": "vq-ope-build-cython-ext-r1-20261009",
+      "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7"
+    },
+    {
+      "arm": "operations",
+      "repeat": 2,
+      "task": "terminal-bench/build-cython-ext",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
+      "job_name": "vq-ope-build-cython-ext-r2-20261009",
+      "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7"
+    },
+    {
+      "arm": "verification",
+      "repeat": 2,
+      "task": "terminal-bench/build-cython-ext",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
+      "job_name": "vq-ver-build-cython-ext-r2-20261009",
+      "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7"
+    },
+    {
+      "arm": "control",
+      "repeat": 2,
+      "task": "terminal-bench/build-cython-ext",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
+      "job_name": "vq-con-build-cython-ext-r2-20261009",
+      "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7"
+    },
+    {
+      "arm": "verification",
+      "repeat": 1,
+      "task": "terminal-bench/cancel-async-tasks",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-ver-cancel-async-tasks-r1-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02"
+    },
+    {
+      "arm": "operations",
+      "repeat": 1,
+      "task": "terminal-bench/cancel-async-tasks",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-ope-cancel-async-tasks-r1-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02"
+    },
+    {
+      "arm": "control",
+      "repeat": 1,
+      "task": "terminal-bench/cancel-async-tasks",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-con-cancel-async-tasks-r1-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02"
+    },
+    {
+      "arm": "control",
+      "repeat": 2,
+      "task": "terminal-bench/cancel-async-tasks",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-con-cancel-async-tasks-r2-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02"
+    },
+    {
+      "arm": "operations",
+      "repeat": 2,
+      "task": "terminal-bench/cancel-async-tasks",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-ope-cancel-async-tasks-r2-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02"
+    },
+    {
+      "arm": "verification",
+      "repeat": 2,
+      "task": "terminal-bench/cancel-async-tasks",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-ver-cancel-async-tasks-r2-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02"
+    }
+  ],
+  "whole_trial_retries": 0,
+  "verifier_policy": "Only timeouts retry; at most three attempts with the original task limit and same agent output.",
+  "preflight_policy": "Run all control task installations and one installation smoke check for each candidate before any model work. No selected task has a registered verifier dependency profile; record not_configured. Full trials repeat setup. Stop before model work if an install-only check fails; preserve the failure.",
+  "freeze_policy": "Only HEADLESS_SYSTEM_PROMPT differs between arms. Runtime, tools, budgets, adapter, task images, and tests remain unchanged. Verify source refs, prompt hashes, and all recorded task-file hashes before launch. Never adjust prompts after outcomes arrive.",
+  "selection_note": "Two diagnostic tasks come from prior observed failures or tool-timeout behavior. Two tasks outside pilot20 were selected by catalog names and public requirements. Candidate prompt text was drafted before reading the new task instructions. Their hidden tests and solutions are not inspected before outputs freeze.",
+  "prior_exposure": "The experiment designer previously inspected pipeline evaluation code during post hoc diagnosis. No task-specific function names, model settings, backward scheduling rule, or hidden expected answer enters either candidate prompt. Pipeline is diagnostic, not held-out.",
+  "primary_metrics": [
+    "eligible official passes, scored failures, setup failures, and verifier errors by task/repeat/arm",
+    "real-entry verification supported by tool output",
+    "critical checks retain item coverage and independently justified expectations",
+    "unresolved failed checks or unsupported final success claims"
+  ],
+  "secondary_metrics": [
+    "estimated cost and accounting completeness",
+    "agent execution seconds, model calls, tool timeouts, truncations and recovery",
+    "stale verification after relevant final edits",
+    "self-authored sensitivity checks where observed"
+  ],
+  "audit_policy": "Review all trajectories for external task-specific solutions/hidden-test exposure. Preserve raw scores but exclude exposed trials from independent-capability comparisons; compare arms on common eligible cases. Evidence absent from a trace remains unknown, not successful. Redact credentials from any published excerpts.",
+  "decision_rule": "Report both candidates separately against the fresh shared control. No outcome-based replacements or extra runs. Two repeats and two held-out tasks are exploratory, not statistically conclusive. Do not promote a candidate based only on pipeline or a single favorable result. Report every regression and cost increase.",
+  "log_dir": "jobs/verification-quality-20261009-orchestration"
+}
diff --git a/benchmarks/harbor/configs/verification-quality-20261009/install-only.json b/benchmarks/harbor/configs/verification-quality-20261009/install-only.json
new file mode 100644
index 0000000..200afe6
--- /dev/null
+++ b/benchmarks/harbor/configs/verification-quality-20261009/install-only.json
@@ -0,0 +1,34 @@
+{
+  "n_concurrent_trials": 2,
+  "n_attempts": 1,
+  "agents": [
+    {
+      "name": "harbor_adapter:NanoPyCodeAgent",
+      "model_name": "deepseek/deepseek-flash",
+      "override_setup_timeout_sec": 1800,
+      "kwargs": {
+        "max_turns": 100,
+        "max_tokens": 65536,
+        "time_budget_seconds": 720
+      }
+    }
+  ],
+  "tasks": [
+    {
+      "name": "terminal-bench/torch-pipeline-parallelism",
+      "ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb"
+    },
+    {
+      "name": "terminal-bench/llm-inference-batching-scheduler",
+      "ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67"
+    },
+    {
+      "name": "terminal-bench/build-cython-ext",
+      "ref": "sha256:35dc561790e73f13e59837828ac0863c0893edc6b71d42cf7d58656c843adf12"
+    },
+    {
+      "name": "terminal-bench/cancel-async-tasks",
+      "ref": "sha256:a3d048d351136e48070696cda8bb79660dfd74db1fea3b6da88559f0332699c1"
+    }
+  ]
+}
diff --git a/benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json b/benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json
new file mode 100644
index 0000000..64469e8
--- /dev/null
+++ b/benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json
@@ -0,0 +1,22 @@
+{
+  "n_concurrent_trials": 1,
+  "n_attempts": 1,
+  "agents": [
+    {
+      "name": "harbor_adapter:NanoPyCodeAgent",
+      "model_name": "deepseek/deepseek-flash",
+      "override_setup_timeout_sec": 1800,
+      "kwargs": {
+        "max_turns": 100,
+        "max_tokens": 65536,
+        "time_budget_seconds": 1620
+      }
+    }
+  ],
+  "tasks": [
+    {
+      "name": "terminal-bench/llm-inference-batching-scheduler",
+      "ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67"
+    }
+  ]
+}
diff --git a/benchmarks/harbor/configs/verification-quality-20261009/operations-guidance.txt b/benchmarks/harbor/configs/verification-quality-20261009/operations-guidance.txt
new file mode 100644
index 0000000..30dd4fe
--- /dev/null
+++ b/benchmarks/harbor/configs/verification-quality-20261009/operations-guidance.txt
@@ -0,0 +1,2 @@
+Each bash call has a 120-second maximum, further limited by the remaining task budget. A longer timeout inside the shell cannot extend it. For longer work, redirect the entire background command group, including stdin, so no child holds the captured pipes; for example: (command; echo $? > /tmp/work.status) > /tmp/work.log 2>&1 < /dev/null &. Use fresh paths for each launch. Poll the log and completion status, inspect the exit code, and validate the resulting artifact before claiming completion. Save a runnable intermediate deliverable before expensive work. Required ongoing services are different from unfinished artifact generation.
+
diff --git a/benchmarks/harbor/configs/verification-quality-20261009/operations-prompt.txt b/benchmarks/harbor/configs/verification-quality-20261009/operations-prompt.txt
new file mode 100644
index 0000000..35652d2
--- /dev/null
+++ b/benchmarks/harbor/configs/verification-quality-20261009/operations-prompt.txt
@@ -0,0 +1,11 @@
+You are nanoPyCodeAgent, a coding agent running non-interactively on a task handed to you up front. There is no user to reply to you: never ask a clarifying question, never stop to wait for confirmation, and never present a plan for approval — decide on your own and carry it out. Work the task through to the end, then check the result with the tools instead of assuming it worked. When it is done, answer with a short summary and no further tool calls: that reply is what ends the run. Runtime budget reminders report remaining time and model replies; when either is running low, prioritize saving the required deliverables, necessary verification, and a final summary. Do not start optional work after the requirements are satisfied.
+
+Identify the required deliverables, interfaces, and constraints. Treat examples as evidence, not as the full contract unless the task says so. Inspect relevant files, dependencies, and existing checks instead of guessing. Task-supplied workspace files, including source, executables, and tests, are available references unless the task restricts their use. Consult general documentation as needed. Do not seek or use task-specific reference solutions or hidden evaluation tests from outside the supplied materials unless the task explicitly requests them. Derive checks from the stated requirements.
+
+Verify the requested behavior through the real entry point when feasible. Check the content and meaning of outputs, not only command success or file existence. For logic or numerical work, choose a few representative and boundary cases justified by the task, using an independent reference, invariant, or consistency check when available. Do not derive every expectation from the implementation being tested or weaken requirements to make a check pass. Keep verification proportional to the task and remaining budget.
+
+When a check fails, inspect the error and relevant output, revise the underlying assumption or implementation, and rerun the affected check. Keep the task's constraints intact. Once appropriate checks support the required behavior, save the deliverables and finish. State what was verified and any remaining limitation without claiming unobserved success.
+
+Each bash call has a 120-second maximum, further limited by the remaining task budget. A longer timeout inside the shell cannot extend it. For longer work, redirect the entire background command group, including stdin, so no child holds the captured pipes; for example: (command; echo $? > /tmp/work.status) > /tmp/work.log 2>&1 < /dev/null &. Use fresh paths for each launch. Poll the log and completion status, inspect the exit code, and validate the resulting artifact before claiming completion. Save a runnable intermediate deliverable before expensive work. Required ongoing services are different from unfinished artifact generation.
+
+Prefer the read tool for viewing files, the edit tool for changing part of an existing file, and the write tool for creating files or rewriting them whole. Use the bash tool to run commands, search with grep, and complete tasks that need real command output instead of guessing.
\ No newline at end of file
diff --git a/benchmarks/harbor/configs/verification-quality-20261009/operations.patch b/benchmarks/harbor/configs/verification-quality-20261009/operations.patch
new file mode 100644
index 0000000..893eb9a
--- /dev/null
+++ b/benchmarks/harbor/configs/verification-quality-20261009/operations.patch
@@ -0,0 +1,12 @@
+diff --git a/src/nanopycodeagent/agent.py b/src/nanopycodeagent/agent.py
+index 4748125..b90b3bf 100644
+--- a/src/nanopycodeagent/agent.py
++++ b/src/nanopycodeagent/agent.py
+@@ -229,6 +229,7 @@ HEADLESS_SYSTEM_PROMPT = (
+     "required behavior, save the deliverables and finish. State what was "
+     "verified and any remaining limitation without claiming unobserved "
+     "success.\n\n"
++    'Each bash call has a 120-second maximum, further limited by the remaining task budget. A longer timeout inside the shell cannot extend it. For longer work, redirect the entire background command group, including stdin, so no child holds the captured pipes; for example: (command; echo $? > /tmp/work.status) > /tmp/work.log 2>&1 < /dev/null &. Use fresh paths for each launch. Poll the log and completion status, inspect the exit code, and validate the resulting artifact before claiming completion. Save a runnable intermediate deliverable before expensive work. Required ongoing services are different from unfinished artifact generation.\n\n'
+ ) + _TOOL_GUIDANCE
+ 
+ def _json_value(value: object) -> JsonValue:
diff --git a/benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json b/benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json
new file mode 100644
index 0000000..9c8a8f9
--- /dev/null
+++ b/benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json
@@ -0,0 +1,22 @@
+{
+  "n_concurrent_trials": 1,
+  "n_attempts": 1,
+  "agents": [
+    {
+      "name": "harbor_adapter:NanoPyCodeAgent",
+      "model_name": "deepseek/deepseek-flash",
+      "override_setup_timeout_sec": 1800,
+      "kwargs": {
+        "max_turns": 100,
+        "max_tokens": 65536,
+        "time_budget_seconds": 720
+      }
+    }
+  ],
+  "tasks": [
+    {
+      "name": "terminal-bench/torch-pipeline-parallelism",
+      "ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb"
+    }
+  ]
+}
diff --git a/benchmarks/harbor/configs/verification-quality-20261009/verification-prompt.txt b/benchmarks/harbor/configs/verification-quality-20261009/verification-prompt.txt
new file mode 100644
index 0000000..eb2879e
--- /dev/null
+++ b/benchmarks/harbor/configs/verification-quality-20261009/verification-prompt.txt
@@ -0,0 +1,11 @@
+You are nanoPyCodeAgent, a coding agent running non-interactively on a task handed to you up front. There is no user to reply to you: never ask a clarifying question, never stop to wait for confirmation, and never present a plan for approval — decide on your own and carry it out. Work the task through to the end, then check the result with the tools instead of assuming it worked. When it is done, answer with a short summary and no further tool calls: that reply is what ends the run. Runtime budget reminders report remaining time and model replies; when either is running low, prioritize saving the required deliverables, necessary verification, and a final summary. Do not start optional work after the requirements are satisfied.
+
+Identify the required deliverables, interfaces, and constraints. Treat examples as evidence, not as the full contract unless the task says so. Inspect relevant files, dependencies, and existing checks instead of guessing. Task-supplied workspace files, including source, executables, and tests, are available references unless the task restricts their use. Consult general documentation as needed. Do not seek or use task-specific reference solutions or hidden evaluation tests from outside the supplied materials unless the task explicitly requests them. Derive checks from the stated requirements.
+
+Before implementing, identify a few high-risk requirements and how to check them against task-provided evidence, an independently computed expectation, or an invariant. Keep this brief. Prefer a small case through the real dependency and entry point over a substitute model. Choose representative and boundary cases justified by the task. For multiple items, preserve their identities and check every item in the chosen case; aggregate or last-item agreement can hide errors. Check observable behavior required by the task as well as final values, without inventing requirements. Keep verification proportional to risk and remaining budget.
+
+When a check fails, distinguish an implementation error, a test error, and an ambiguous requirement before changing code or expectations. Changing an expected value, tolerance, selection, or ordering needs support from the requirements or an independent reference; retain coverage of the original failing behavior. Do not make the test merely mirror the implementation. For a critical self-authored check, when inexpensive, confirm that it rejects a deliberately incorrect result. Repair the cause and rerun affected checks.
+
+Before finishing, inspect the actual check results for the final saved deliverables. A successful command exit or file existence alone does not establish correct behavior. Identify unresolved failures and gaps where only a substitute was checked; do not claim those requirements passed. Once appropriate checks support the required behavior, finish with the observed evidence and remaining limitations.
+
+Prefer the read tool for viewing files, the edit tool for changing part of an existing file, and the write tool for creating files or rewriting them whole. Use the bash tool to run commands, search with grep, and complete tasks that need real command output instead of guessing.
\ No newline at end of file
diff --git a/benchmarks/harbor/configs/verification-quality-20261009/verification.patch b/benchmarks/harbor/configs/verification-quality-20261009/verification.patch
new file mode 100644
index 0000000..c62dd50
--- /dev/null
+++ b/benchmarks/harbor/configs/verification-quality-20261009/verification.patch
@@ -0,0 +1,48 @@
+diff --git a/src/nanopycodeagent/agent.py b/src/nanopycodeagent/agent.py
+index 4748125..3cae8ca 100644
+--- a/src/nanopycodeagent/agent.py
++++ b/src/nanopycodeagent/agent.py
+@@ -215,20 +215,29 @@ HEADLESS_SYSTEM_PROMPT = (
+     "reference solutions or hidden evaluation tests from outside the supplied "
+     "materials unless the task explicitly requests them. Derive checks from "
+     "the stated requirements.\n\n"
+-    "Verify the requested behavior through the real entry point when feasible. "
+-    "Check the content and meaning of outputs, not only command success or "
+-    "file existence. For logic or numerical work, choose a few representative "
+-    "and boundary cases justified by the task, using an independent reference, "
+-    "invariant, or consistency check when available. Do not derive every "
+-    "expectation from the implementation being tested or weaken requirements "
+-    "to make a check pass. Keep verification proportional to the task and "
+-    "remaining budget.\n\n"
+-    "When a check fails, inspect the error and relevant output, revise the "
+-    "underlying assumption or implementation, and rerun the affected check. "
+-    "Keep the task's constraints intact. Once appropriate checks support the "
+-    "required behavior, save the deliverables and finish. State what was "
+-    "verified and any remaining limitation without claiming unobserved "
+-    "success.\n\n"
++    "Before implementing, identify a few high-risk requirements and how to "
++    "check them against task-provided evidence, an independently computed "
++    "expectation, or an invariant. Keep this brief. Prefer a small case through "
++    "the real dependency and entry point over a substitute model. Choose "
++    "representative and boundary cases justified by the task. For multiple "
++    "items, preserve their identities and check every item in the chosen case; "
++    "aggregate or last-item agreement can hide errors. Check observable "
++    "behavior required by the task as well as final values, without inventing "
++    "requirements. Keep verification proportional to risk and remaining budget.\n\n"
++    "When a check fails, distinguish an implementation error, a test error, "
++    "and an ambiguous requirement before changing code or expectations. "
++    "Changing an expected value, tolerance, selection, or ordering needs "
++    "support from the requirements or an independent reference; retain "
++    "coverage of the original failing behavior. Do not make the test merely "
++    "mirror the implementation. For a critical self-authored check, when "
++    "inexpensive, confirm that it rejects a deliberately incorrect result. "
++    "Repair the cause and rerun affected checks.\n\n"
++    "Before finishing, inspect the actual check results for the final saved "
++    "deliverables. A successful command exit or file existence alone does "
++    "not establish correct behavior. Identify unresolved failures and gaps "
++    "where only a substitute was checked; do not claim those requirements "
++    "passed. Once appropriate checks support the required behavior, finish "
++    "with the observed evidence and remaining limitations.\n\n"
+ ) + _TOOL_GUIDANCE
+ 
+ def _json_value(value: object) -> JsonValue:
diff --git a/benchmarks/harbor/scripts/run_comparison.py b/benchmarks/harbor/scripts/run_comparison.py
new file mode 100644
index 0000000..a9629aa
--- /dev/null
+++ b/benchmarks/harbor/scripts/run_comparison.py
@@ -0,0 +1,150 @@
+"""Run a frozen comparison plan through the standard Harbor entry point.
+
+This orchestrates existing repository worktrees; it does not install dependencies,
+modify task environments, implement an adapter, or retry a trial.
+"""
+
+import argparse
+import ast
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timezone
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+
+
+def digest(path):
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def prompt_from_source(path):
+    values = {}
+    for node in ast.parse(path.read_text()).body:
+        if isinstance(node, ast.Assign) and len(node.targets) == 1:
+            target = node.targets[0]
+            if isinstance(target, ast.Name) and target.id in {
+                "_TOOL_GUIDANCE", "HEADLESS_SYSTEM_PROMPT",
+            }:
+                values[target.id] = eval(
+                    compile(ast.Expression(node.value), str(path), "eval"),
+                    {"__builtins__": {}}, values,
+                )
+    return values["HEADLESS_SYSTEM_PROMPT"]
+
+
+def validate_plan(root, plan_path, plan):
+    jobs = plan["preflight"] + plan["queue"]
+    names = [job["job_name"] for job in jobs]
+    if len(names) != len(set(names)):
+        raise ValueError("Job names must be unique")
+    if len(plan["queue"]) != plan["total_model_runs"]:
+        raise ValueError("Model run count does not match the registered queue")
+    if plan["max_concurrent_trials"] != 2 or plan["whole_trial_retries"] != 0:
+        raise ValueError("This runner requires concurrency two and no trial retries")
+    for arm in plan["arms"].values():
+        source = root / arm["source_root"]
+        actual_ref = subprocess.check_output(
+            ["git", "rev-parse", "HEAD"], cwd=source, text=True,
+        ).strip()
+        if actual_ref != arm["source_ref"]:
+            raise ValueError(f"Source revision changed: {source}")
+        subprocess.run(
+            ["git", "diff", "--exit-code", "HEAD", "--", "src", "benchmarks/harbor/src"],
+            cwd=source, check=True, stdout=subprocess.DEVNULL,
+        )
+        expected = plan_path.parent / arm["prompt_file"]
+        if digest(expected) != arm["prompt_sha256"]:
+            raise ValueError(f"Prompt file changed: {expected}")
+        if prompt_from_source(source / "src/nanopycodeagent/agent.py") != expected.read_text():
+            raise ValueError(f"Source prompt does not match the plan: {source}")
+    for task in plan["tasks"]:
+        cache = Path.home() / ".cache/harbor/tasks/packages" / task["name"] / task["ref"].split(":")[1]
+        for name, expected in task["task_file_sha256"].items():
+            if digest(cache / name) != expected:
+                raise ValueError(f"Task file changed: {cache / name}")
+    for job in jobs:
+        config_path = root / job["config"]
+        if digest(config_path) != job["config_sha256"]:
+            raise ValueError(f"Configuration changed: {config_path}")
+        config = json.loads(config_path.read_text())
+        limit = 1 if job in plan["queue"] else 2
+        if config["n_concurrent_trials"] > limit or config["n_attempts"] != 1:
+            raise ValueError("Per-job concurrency or attempts exceed the plan")
+
+
+def run_job(root, plan, job, record, install_only):
+    status_path = record / (job["job_name"] + ".json")
+    if status_path.exists():
+        previous = json.loads(status_path.read_text())
+        if previous["status"] == "finished":
+            return previous
+        raise RuntimeError(f"Previously started job requires inspection; refusing to retry: {status_path}")
+    source = root / plan["arms"][job["arm"]]["source_root"]
+    command = [
+        "uv", "run", "--project", "benchmarks/harbor", "python", "-m", "harbor_adapter",
+        "run", "--config", str(root / job["config"]), "--job-name", job["job_name"],
+        "--cache-dir", str(root / ".cache/nanopy-harbor"),
+    ]
+    if install_only:
+        command.append("--install-only")
+    summary_path = source / "jobs" / (job["job_name"] + "-input") / "summary.json"
+    state = {
+        **job, "status": "started", "started_at": datetime.now(timezone.utc).isoformat(),
+        "command": command, "cwd": str(source), "summary_path": str(summary_path),
+    }
+    status_path.write_text(json.dumps(state, indent=2) + "\n")
+    with (record / (job["job_name"] + ".log")).open("w") as log:
+        code = subprocess.run(command, cwd=source, stdout=log, stderr=subprocess.STDOUT).returncode
+    state.update(status="finished", exit_code=code, finished_at=datetime.now(timezone.utc).isoformat())
+    if summary_path.exists():
+        state["summary"] = json.loads(summary_path.read_text())
+    status_path.write_text(json.dumps(state, indent=2) + "\n")
+    print(json.dumps({"job": job["job_name"], "exit_code": code, "status": state["status"]}), flush=True)
+    return state
+
+
+def preflight_passed(states):
+    return bool(states) and all(
+        s.get("exit_code") == 0
+        and s.get("summary", {}).get("trials")
+        and all(t["status"] == "setup_passed" for t in s["summary"]["trials"])
+        for s in states
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--plan", type=Path, required=True)
+    parser.add_argument("--phase", choices=["preflight", "model"], required=True)
+    args = parser.parse_args()
+    root, plan_path = Path.cwd(), args.plan.resolve()
+    plan = json.loads(plan_path.read_text())
+    validate_plan(root, plan_path, plan)
+    record = root / plan["log_dir"]
+    record.mkdir(parents=True, exist_ok=True)
+    record.chmod(0o700)
+    registration = record / "registration.json"
+    current = {"plan_sha256": digest(plan_path), "plan": plan}
+    if registration.exists():
+        if json.loads(registration.read_text()) != current:
+            raise ValueError("The registered plan is immutable")
+    else:
+        registration.write_text(json.dumps(current, indent=2) + "\n")
+    if args.phase == "preflight":
+        states = [run_job(root, plan, j, record, True) for j in plan["preflight"]]
+        return 0 if preflight_passed(states) else 1
+    preflight = [json.loads((record / (j["job_name"] + ".json")).read_text()) for j in plan["preflight"]]
+    if not preflight_passed(preflight):
+        raise RuntimeError("Install-only checks must finish successfully before model work")
+    states = []
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        for offset in range(0, len(plan["queue"]), 2):
+            batch = plan["queue"][offset:offset + 2]
+            futures = [pool.submit(run_job, root, plan, job, record, False) for job in batch]
+            states.extend(f.result() for f in futures)
+    return int(any(s["exit_code"] != 0 for s in states))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/benchmarks/harbor/tests/test_comparison.py b/benchmarks/harbor/tests/test_comparison.py
new file mode 100644
index 0000000..da0db25
--- /dev/null
+++ b/benchmarks/harbor/tests/test_comparison.py
@@ -0,0 +1,55 @@
+"""Comparison orchestration must preserve failed trials and preflight gates."""
+
+import importlib.util
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+
+spec = importlib.util.spec_from_file_location(
+    "run_comparison", Path(__file__).parents[1] / "scripts/run_comparison.py",
+)
+comparison = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(comparison)
+
+
+@pytest.mark.parametrize("states", [
+    [], [{"exit_code": 0}],
+    [{"exit_code": 0, "summary": {"trials": []}}],
+    [{"exit_code": 1, "summary": {"trials": [{"status": "setup_passed"}]}}],
+    [{"exit_code": 0, "summary": {"trials": [{"status": "setup_failed"}]}}],
+])
+def test_incomplete_or_failed_preflight_cannot_start_models(states):
+    assert not comparison.preflight_passed(states)
+
+
+def test_successful_installation_does_not_require_a_registered_profile():
+    state = {"exit_code": 0, "summary": {"trials": [{
+        "status": "setup_passed", "bootstrap": {"verifier_preflight": {"status": "not_configured"}},
+    }]}}
+    assert comparison.preflight_passed([state])
+
+
+def test_failed_job_is_recorded_and_never_retried(tmp_path, monkeypatch):
+    plan = {"arms": {"control": {"source_root": "source"}}}
+    job = {"arm": "control", "config": "config.json", "job_name": "test-job"}
+    calls = []
+    def run(command, **kwargs):
+        calls.append(command)
+        return SimpleNamespace(returncode=1)
+    monkeypatch.setattr(comparison.subprocess, "run", run)
+    first = comparison.run_job(tmp_path, plan, job, tmp_path, False)
+    second = comparison.run_job(tmp_path, plan, job, tmp_path, False)
+    assert first == second
+    assert first["exit_code"] == 1
+    assert len(calls) == 1
+    assert calls[0][:8] == ["uv", "run", "--project", "benchmarks/harbor", "python", "-m", "harbor_adapter", "run"]
+
+
+def test_interrupted_job_cannot_silently_restart(tmp_path):
+    job = {"job_name": "interrupted"}
+    (tmp_path / "interrupted.json").write_text(json.dumps({"status": "started"}))
+    with pytest.raises(RuntimeError, match="refusing to retry"):
+        comparison.run_job(tmp_path, {}, job, tmp_path, False)

From 518a0709a71b06efbaa3478b46bb9554273b61b4 Mon Sep 17 00:00:00 2001
From: minixalpha <xkzalpha@gmail.com>
Date: Fri, 9 Oct 2026 07:47:30 -0700
Subject: [PATCH 3/7] fix(benchmarks): require every planned preflight result

---
 benchmarks/harbor/scripts/run_comparison.py | 15 ++++++++++-----
 benchmarks/harbor/tests/test_comparison.py  |  6 ++++--
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/benchmarks/harbor/scripts/run_comparison.py b/benchmarks/harbor/scripts/run_comparison.py
index a9629aa..9266f1f 100644
--- a/benchmarks/harbor/scripts/run_comparison.py
+++ b/benchmarks/harbor/scripts/run_comparison.py
@@ -104,12 +104,13 @@ def run_job(root, plan, job, record, install_only):
     return state
 
 
-def preflight_passed(states):
-    return bool(states) and all(
+def preflight_passed(states, expected_counts):
+    return bool(states) and len(states) == len(expected_counts) and all(
         s.get("exit_code") == 0
         and s.get("summary", {}).get("trials")
+        and len(s["summary"]["trials"]) == expected
         and all(t["status"] == "setup_passed" for t in s["summary"]["trials"])
-        for s in states
+        for s, expected in zip(states, expected_counts)
     )
 
 
@@ -131,11 +132,15 @@ def main():
             raise ValueError("The registered plan is immutable")
     else:
         registration.write_text(json.dumps(current, indent=2) + "\n")
+    expected_counts = [
+        len(json.loads((root / job["config"]).read_text())["tasks"])
+        for job in plan["preflight"]
+    ]
     if args.phase == "preflight":
         states = [run_job(root, plan, j, record, True) for j in plan["preflight"]]
-        return 0 if preflight_passed(states) else 1
+        return 0 if preflight_passed(states, expected_counts) else 1
     preflight = [json.loads((record / (j["job_name"] + ".json")).read_text()) for j in plan["preflight"]]
-    if not preflight_passed(preflight):
+    if not preflight_passed(preflight, expected_counts):
         raise RuntimeError("Install-only checks must finish successfully before model work")
     states = []
     with ThreadPoolExecutor(max_workers=2) as pool:
diff --git a/benchmarks/harbor/tests/test_comparison.py b/benchmarks/harbor/tests/test_comparison.py
index da0db25..61faa56 100644
--- a/benchmarks/harbor/tests/test_comparison.py
+++ b/benchmarks/harbor/tests/test_comparison.py
@@ -22,14 +22,16 @@
     [{"exit_code": 0, "summary": {"trials": [{"status": "setup_failed"}]}}],
 ])
 def test_incomplete_or_failed_preflight_cannot_start_models(states):
-    assert not comparison.preflight_passed(states)
+    assert not comparison.preflight_passed(states, [1] * len(states))
 
 
 def test_successful_installation_does_not_require_a_registered_profile():
     state = {"exit_code": 0, "summary": {"trials": [{
         "status": "setup_passed", "bootstrap": {"verifier_preflight": {"status": "not_configured"}},
     }]}}
-    assert comparison.preflight_passed([state])
+    assert comparison.preflight_passed([state], [1])
+    assert not comparison.preflight_passed([state], [4])
+    assert not comparison.preflight_passed([state], [1, 1])
 
 
 def test_failed_job_is_recorded_and_never_retried(tmp_path, monkeypatch):

From 4b7b02f2cc6f45bd286a67575ce19eefcc0c8db5 Mon Sep 17 00:00:00 2001
From: minixalpha <xkzalpha@gmail.com>
Date: Fri, 9 Oct 2026 08:38:22 -0700
Subject: [PATCH 4/7] docs(benchmarks): record initial verification comparison
 evidence

---
 .../reports/verification-quality-20261009.md  |  94 ++++
 .../verification-quality-20261009.json        | 473 ++++++++++++++++++
 .../verification-quality-audit-20261009.json  |  67 +++
 ...rification-quality-preflight-20261009.json | 105 ++++
 4 files changed, 739 insertions(+)
 create mode 100644 benchmarks/harbor/reports/verification-quality-20261009.md
 create mode 100644 benchmarks/harbor/results/verification-quality-20261009.json
 create mode 100644 benchmarks/harbor/results/verification-quality-audit-20261009.json
 create mode 100644 benchmarks/harbor/results/verification-quality-preflight-20261009.json

diff --git a/benchmarks/harbor/reports/verification-quality-20261009.md b/benchmarks/harbor/reports/verification-quality-20261009.md
new file mode 100644
index 0000000..7bd4754
--- /dev/null
+++ b/benchmarks/harbor/reports/verification-quality-20261009.md
@@ -0,0 +1,94 @@
+# Verification quality and long-command guidance comparison
+
+**Status: running.** This report is incomplete and makes no capability or
+efficiency improvement claim. The fixed queue contains 24 model trials; the
+first completed verification-candidate pipeline trial scored zero. The matching
+control has finished model work but is still awaiting an official result after
+two verifier attempts timed out during dependency installation; the third and
+last permitted attempt is running.
+
+The [registered plan](../configs/verification-quality-20261009/experiment.json)
+and [reproduction instructions](../configs/verification-quality-20261009/README.md)
+freeze three variants, four task revisions, and two repeats per task and arm.
+The [structured results](../results/verification-quality-20261009.json) retain
+pending outcomes separately from completed scores. The [trajectory audit](../results/verification-quality-audit-20261009.json)
+is manual and unblinded; its current entries cover only completed agent runs.
+
+## Frozen variants
+
+| Arm | Source | Prompt words | Change |
+| --- | --- | ---: | --- |
+| Control | `2eb99b608dfb39464d17d109a4a8f11afda607a9` | 385 | Main with the task-supplied-reference clarification |
+| Verification | `3b69d6d` | 482 | Concrete evidence selection, real-entry checks, item coverage, justified test corrections, and final-version evidence review |
+| Operations | `ca624b0` | 480 | Explicit Bash limits, redirected background commands, completion status, and intermediate deliverables |
+
+Only `HEADLESS_SYSTEM_PROMPT` differs in the frozen runtime sources. Removing
+that assignment produces identical agent ASTs; all other runtime/adapter Python
+files are unchanged. The primary branch also contains experiment orchestration,
+but every trial is built by the standard runner from its isolated frozen source.
+
+All arms use the official DeepSeek endpoint with `deepseek/deepseek-flash`,
+100 replies, and 65,536 output tokens per response. Temperature and reasoning
+effort are not explicitly configured. At most two trials run concurrently.
+Whole-trial retries are disabled. Only verifier timeouts retry, at most three
+attempts with the same agent output and original per-attempt deadline.
+
+## Scope and setup
+
+The diagnostic tasks are `torch-pipeline-parallelism` (720 seconds of model
+work) and `llm-inference-batching-scheduler` (1,620 seconds). The two tasks outside
+the previous pilot20 tuning cohort are `build-cython-ext` and
+`cancel-async-tasks` (720 seconds each). Each task runs twice in each arm.
+This is an exploratory subset comparison, not a new twenty-task score.
+
+The experiment designer previously inspected pipeline evaluation code during
+post hoc diagnosis. Neither candidate receives a task-specific backward-order
+rule, hidden expected answer, function name, or model configuration. Candidate
+text was drafted before reading the new tasks' public instructions. Their hidden
+tests and solutions remain uninspected until every scheduled output is fixed.
+
+[Install-only evidence](../results/verification-quality-preflight-20261009.json)
+records four successful control task installations and one successful wheel
+installation for each candidate. All four verifier-dependency profiles are
+explicitly `not_configured`; these installation checks do not certify official
+verifier dependencies. The original task image IDs are recorded, and both
+diagnostic images match the prior pilot. Task files and official scripts remain
+unchanged.
+
+## Initial observations, pending the remaining trials
+
+The first control pipeline trajectory exercised a small real Llama model. Its
+single-rank check compared all selected microbatch activations and 39 parameter
+gradients against a full-batch reference through the model's own forward path.
+That check passed. Two-rank checks timed out or produced no completion output;
+the final response disclosed this gap. Its network requests retrieved general
+Transformers/PyTorch source and packages, with no task-answer retrieval observed.
+
+The first verification-candidate pipeline trajectory saved an implementation and
+passed `py_compile`, but could not import Torch. It wrote a numerical harness
+without executing it. Two Bash commands hit the 120-second tool limit, including
+one with an inner `timeout 300`. The final response explicitly disclosed that
+runtime behavior was unverified. Exact-function searches returned authentication
+errors, missing endpoints, security challenges, or no answer links; no task-answer
+content was observed. These attempts are recorded even without confirmed exposure.
+
+The candidate's official verifier completed with reward zero: the single-rank
+case failed a backward-activation comparison (`lm_head.bwd`, maximum difference
+approximately 0.5714225), while the two-rank case raised `AttributeError` because
+`local_targets[micro]` was `None`. The implementation chose multiplication by the
+microbatch count and reverse-order backward processing. This output had no
+successful numerical self-check. The failure is retained without a replacement
+model run.
+
+These two trajectories do not show improved verification coverage from the
+expanded verification prompt. They also do not establish a general regression:
+the remaining repetitions and tasks have not finished, dependency-download
+conditions differed, and the control's official result is still pending.
+
+## Local and CI validation
+
+Core tests: **351 passed, 4 skipped**. Harbor workflow tests: **99 passed**.
+The eight orchestration tests passed again after tightening the preflight gate
+to require every planned installation result. Draft PR #48 passed the Python
+3.13, Python 3.14, and Harbor workflow CI checks. Research notes are unchanged in
+both languages.
diff --git a/benchmarks/harbor/results/verification-quality-20261009.json b/benchmarks/harbor/results/verification-quality-20261009.json
new file mode 100644
index 0000000..1e28208
--- /dev/null
+++ b/benchmarks/harbor/results/verification-quality-20261009.json
@@ -0,0 +1,473 @@
+{
+  "schema_version": 1,
+  "generated_at": "2026-10-09T15:37:51.389849+00:00",
+  "experiment_plan": "benchmarks/harbor/configs/verification-quality-20261009/experiment.json",
+  "experiment_plan_sha256": "b3968a10a29507b72c62f6f886a6af4f982373e15ca6103da21dd288362396c1",
+  "status": "running",
+  "summary": {
+    "control": {
+      "planned": 8,
+      "statuses": {
+        "verifying": 1,
+        "not_started": 7
+      },
+      "passes": 0,
+      "scored_failures": 0,
+      "known_cost_usd": 0.085583436,
+      "cost_complete": false,
+      "usage_complete": false,
+      "known_agent_seconds": 0,
+      "model_attempts": 27,
+      "tool_timeouts": 1
+    },
+    "verification": {
+      "planned": 8,
+      "statuses": {
+        "scored": 1,
+        "not_started": 7
+      },
+      "passes": 0,
+      "scored_failures": 1,
+      "known_cost_usd": 0.0612708,
+      "cost_complete": false,
+      "usage_complete": false,
+      "known_agent_seconds": 655.750458,
+      "model_attempts": 20,
+      "tool_timeouts": 2
+    },
+    "operations": {
+      "planned": 8,
+      "statuses": {
+        "not_started": 8
+      },
+      "passes": 0,
+      "scored_failures": 0,
+      "known_cost_usd": 0,
+      "cost_complete": false,
+      "usage_complete": false,
+      "known_agent_seconds": 0,
+      "model_attempts": 0,
+      "tool_timeouts": 0
+    }
+  },
+  "trials": [
+    {
+      "arm": "control",
+      "repeat": 1,
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
+      "job_name": "vq-con-torch-pipeline-parallelism-r1-20261009",
+      "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
+      "status": "verifying",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": 0.085583436,
+      "cost_complete": true,
+      "usage_complete": true,
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__YcctKv5/agent/trajectory.json",
+      "trajectory_sha256": "698e12648ccb5f0f8312d6ee8565d7cec6185f5aa535ab21db847bcb03b08cc2",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 671658.796661,
+        "timestamp": "2026-10-09T15:05:36.416Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "metrics": {
+        "total_steps": 55,
+        "total_prompt_tokens": 1230161,
+        "total_completion_tokens": 56274,
+        "total_cached_tokens": 1193856,
+        "total_cost_usd": 0.085583436,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.085583436
+        }
+      },
+      "model_attempts": 27,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "recovery_events": 0,
+      "tool_timeouts": 1,
+      "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
+      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350"
+    },
+    {
+      "arm": "verification",
+      "repeat": 1,
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
+      "job_name": "vq-ver-torch-pipeline-parallelism-r1-20261009",
+      "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
+      "status": "scored",
+      "reward": 0.0,
+      "launcher_exit_code": 0,
+      "known_cost_usd": 0.0612708,
+      "cost_complete": true,
+      "usage_complete": true,
+      "raw_verifier_rewards": {
+        "reward": 0.0
+      },
+      "exception_type": null,
+      "result_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__TxDd3vH/result.json",
+      "task_checksum": "31e565558cd22598a3c2069f771d9d0eee26fb843b77f7b03e00ad4c336fbce9",
+      "agent_setup_seconds": 77.230504,
+      "agent_execution_seconds": 655.750458,
+      "verifier_seconds": 703.878173,
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "started_at": "2026-10-09T14:52:06.479199Z",
+      "finished_at": "2026-10-09T15:16:18.452768Z",
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 703.7512151589981
+          }
+        ]
+      },
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__TxDd3vH/agent/trajectory.json",
+      "trajectory_sha256": "c47b1502cc633647967ba9b7fbd7a99c9528a35180e7e826a6d6bb76b5f938d8",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 655947.282239,
+        "timestamp": "2026-10-09T15:04:20.413Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "metrics": {
+        "total_steps": 41,
+        "total_prompt_tokens": 600080,
+        "total_completion_tokens": 46079,
+        "total_cached_tokens": 592000,
+        "total_cost_usd": 0.0612708,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.0612708
+        }
+      },
+      "model_attempts": 20,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "recovery_events": 0,
+      "tool_timeouts": 2,
+      "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
+      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c"
+    },
+    {
+      "arm": "operations",
+      "repeat": 1,
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
+      "job_name": "vq-ope-torch-pipeline-parallelism-r1-20261009",
+      "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "operations",
+      "repeat": 2,
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
+      "job_name": "vq-ope-torch-pipeline-parallelism-r2-20261009",
+      "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "verification",
+      "repeat": 2,
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
+      "job_name": "vq-ver-torch-pipeline-parallelism-r2-20261009",
+      "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "control",
+      "repeat": 2,
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
+      "job_name": "vq-con-torch-pipeline-parallelism-r2-20261009",
+      "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "verification",
+      "repeat": 1,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-ver-llm-inference-batching-scheduler-r1-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "operations",
+      "repeat": 1,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-ope-llm-inference-batching-scheduler-r1-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "control",
+      "repeat": 1,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-con-llm-inference-batching-scheduler-r1-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "control",
+      "repeat": 2,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-con-llm-inference-batching-scheduler-r2-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "operations",
+      "repeat": 2,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-ope-llm-inference-batching-scheduler-r2-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "verification",
+      "repeat": 2,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-ver-llm-inference-batching-scheduler-r2-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "control",
+      "repeat": 1,
+      "task": "terminal-bench/build-cython-ext",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
+      "job_name": "vq-con-build-cython-ext-r1-20261009",
+      "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "verification",
+      "repeat": 1,
+      "task": "terminal-bench/build-cython-ext",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
+      "job_name": "vq-ver-build-cython-ext-r1-20261009",
+      "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "operations",
+      "repeat": 1,
+      "task": "terminal-bench/build-cython-ext",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
+      "job_name": "vq-ope-build-cython-ext-r1-20261009",
+      "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "operations",
+      "repeat": 2,
+      "task": "terminal-bench/build-cython-ext",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
+      "job_name": "vq-ope-build-cython-ext-r2-20261009",
+      "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "verification",
+      "repeat": 2,
+      "task": "terminal-bench/build-cython-ext",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
+      "job_name": "vq-ver-build-cython-ext-r2-20261009",
+      "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "control",
+      "repeat": 2,
+      "task": "terminal-bench/build-cython-ext",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
+      "job_name": "vq-con-build-cython-ext-r2-20261009",
+      "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "verification",
+      "repeat": 1,
+      "task": "terminal-bench/cancel-async-tasks",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-ver-cancel-async-tasks-r1-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "operations",
+      "repeat": 1,
+      "task": "terminal-bench/cancel-async-tasks",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-ope-cancel-async-tasks-r1-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "control",
+      "repeat": 1,
+      "task": "terminal-bench/cancel-async-tasks",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-con-cancel-async-tasks-r1-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "control",
+      "repeat": 2,
+      "task": "terminal-bench/cancel-async-tasks",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-con-cancel-async-tasks-r2-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "operations",
+      "repeat": 2,
+      "task": "terminal-bench/cancel-async-tasks",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-ope-cancel-async-tasks-r2-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    },
+    {
+      "arm": "verification",
+      "repeat": 2,
+      "task": "terminal-bench/cancel-async-tasks",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
+      "job_name": "vq-ver-cancel-async-tasks-r2-20261009",
+      "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02",
+      "status": "not_started",
+      "reward": null,
+      "launcher_exit_code": null,
+      "known_cost_usd": null,
+      "cost_complete": false,
+      "usage_complete": false
+    }
+  ],
+  "interpretation": "Raw outcomes. Manual trajectory audits and common-eligibility comparisons are reported separately. Missing costs and pending outcomes are not zero-cost failures."
+}
diff --git a/benchmarks/harbor/results/verification-quality-audit-20261009.json b/benchmarks/harbor/results/verification-quality-audit-20261009.json
new file mode 100644
index 0000000..91adb92
--- /dev/null
+++ b/benchmarks/harbor/results/verification-quality-audit-20261009.json
@@ -0,0 +1,67 @@
+{
+  "schema_version": 1,
+  "status": "in_progress",
+  "method": "Manual, unblinded trajectory review; evidence absence is not evidence of successful validation. No hidden tests for the new tasks are read before all outputs freeze.",
+  "trials": [
+    {
+      "job": "vq-con-torch-pipeline-parallelism-r1-20261009",
+      "arm": "control",
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "repeat": 1,
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__YcctKv5/agent/trajectory.json",
+      "trajectory_sha256": "698e12648ccb5f0f8312d6ee8565d7cec6185f5aa535ab21db847bcb03b08cc2",
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "partial_pass",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            31,
+            49,
+            51,
+            55
+          ],
+          "note": "The real Llama model passes a single-rank check against its full-batch forward, including every selected microbatch and 39 parameter gradients. The two-rank attempts time out or have no completion output. The final explicitly discloses the two-rank gap. Network calls retrieve general Transformers/PyTorch source and packages."
+        }
+      ]
+    },
+    {
+      "job": "vq-ver-torch-pipeline-parallelism-r1-20261009",
+      "arm": "verification",
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "repeat": 1,
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__TxDd3vH/agent/trajectory.json",
+      "trajectory_sha256": "c47b1502cc633647967ba9b7fbd7a99c9528a35180e7e826a6d6bb76b5f938d8",
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": true,
+      "real_entry_check": "not_executed",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            9,
+            11,
+            13,
+            17,
+            19,
+            27,
+            29,
+            37,
+            41
+          ],
+          "note": "Exact-function searches return security challenges, missing endpoints, authentication errors, or no answer links. Two Bash calls time out at 120 seconds. Torch remains unavailable; a real-model harness is written but not executed. Only py_compile passes, and the final explicitly discloses no numerical verification and the chosen loss-scale interpretation."
+        }
+      ],
+      "official_failure_notes": "Completed verifier reward 0: world_size=1 backward activation mismatch at lm_head (max diff 0.5714225); world_size=2 raises AttributeError because local_targets[micro] is None. The candidate chose multiplication by the microbatch count and did not run its numerical harness."
+    }
+  ]
+}
diff --git a/benchmarks/harbor/results/verification-quality-preflight-20261009.json b/benchmarks/harbor/results/verification-quality-preflight-20261009.json
new file mode 100644
index 0000000..9f92fcd
--- /dev/null
+++ b/benchmarks/harbor/results/verification-quality-preflight-20261009.json
@@ -0,0 +1,105 @@
+{
+  "schema_version": 1,
+  "recorded_at": "2026-10-09T14:53:01.919625+00:00",
+  "plan": "benchmarks/harbor/configs/verification-quality-20261009/experiment.json",
+  "install_only": [
+    {
+      "arm": "control",
+      "task": "terminal-bench/build-cython-ext",
+      "status": "setup_passed",
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      }
+    },
+    {
+      "arm": "control",
+      "task": "terminal-bench/cancel-async-tasks",
+      "status": "setup_passed",
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      }
+    },
+    {
+      "arm": "control",
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "status": "setup_passed",
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      }
+    },
+    {
+      "arm": "control",
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "status": "setup_passed",
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      }
+    },
+    {
+      "arm": "verification",
+      "task": "terminal-bench/cancel-async-tasks",
+      "status": "setup_passed",
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      }
+    },
+    {
+      "arm": "operations",
+      "task": "terminal-bench/cancel-async-tasks",
+      "status": "setup_passed",
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      }
+    }
+  ],
+  "images": [
+    {
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "image": "alexgshaw/torch-pipeline-parallelism:20251031",
+      "image_id": "sha256:3cb7b39d86f1704b20e668851da0b08b34b3270c01d583f136d65190984a8d1b",
+      "repo_digests": [
+        "alexgshaw/torch-pipeline-parallelism@sha256:3cb7b39d86f1704b20e668851da0b08b34b3270c01d583f136d65190984a8d1b"
+      ],
+      "size": 29725531
+    },
+    {
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "image": "alexgshaw/llm-inference-batching-scheduler:20251031",
+      "image_id": "sha256:19e79aa49be4e55dccca1dde966515ebe6852314b6010bcb9230afb48b996774",
+      "repo_digests": [
+        "alexgshaw/llm-inference-batching-scheduler@sha256:19e79aa49be4e55dccca1dde966515ebe6852314b6010bcb9230afb48b996774"
+      ],
+      "size": 44188631
+    },
+    {
+      "task": "terminal-bench/build-cython-ext",
+      "image": "alexgshaw/build-cython-ext:20251031",
+      "image_id": "sha256:3612a38fadb89a96f74a1a951fb0b0af734198fd160571eeaba6401593234594",
+      "repo_digests": [
+        "alexgshaw/build-cython-ext@sha256:3612a38fadb89a96f74a1a951fb0b0af734198fd160571eeaba6401593234594"
+      ],
+      "size": 303222492
+    },
+    {
+      "task": "terminal-bench/cancel-async-tasks",
+      "image": "alexgshaw/cancel-async-tasks:20251031",
+      "image_id": "sha256:84c7fae6b256dcc56a350790e2a9715eefc7dad662a9d8e8a472363aa71ef18d",
+      "repo_digests": [
+        "alexgshaw/cancel-async-tasks@sha256:84c7fae6b256dcc56a350790e2a9715eefc7dad662a9d8e8a472363aa71ef18d"
+      ],
+      "size": 44171676
+    }
+  ],
+  "source_checks": {
+    "only_headless_prompt_differs": true,
+    "other_agent_ast_identical": true,
+    "frozen_sources_clean": true
+  },
+  "limitations": "Install-only validates agent installation. No selected task has a registered verifier-dependency profile. These checks do not certify official verifier dependencies."
+}

From 3f8862420ce2a3f125c282b7e82aaa45d127bca2 Mon Sep 17 00:00:00 2001
From: minixalpha <xkzalpha@gmail.com>
Date: Fri, 9 Oct 2026 09:00:44 -0700
Subject: [PATCH 5/7] docs(benchmarks): distinguish verifier errors from scored
 failures

---
 .../reports/verification-quality-20261009.md  |  34 ++-
 .../verification-quality-20261009.json        | 264 ++++++++++++++++--
 .../verification-quality-audit-20261009.json  |  75 ++++-
 3 files changed, 345 insertions(+), 28 deletions(-)

diff --git a/benchmarks/harbor/reports/verification-quality-20261009.md b/benchmarks/harbor/reports/verification-quality-20261009.md
index 7bd4754..3823858 100644
--- a/benchmarks/harbor/reports/verification-quality-20261009.md
+++ b/benchmarks/harbor/reports/verification-quality-20261009.md
@@ -1,11 +1,11 @@
 # Verification quality and long-command guidance comparison
 
-**Status: running.** This report is incomplete and makes no capability or
-efficiency improvement claim. The fixed queue contains 24 model trials; the
-first completed verification-candidate pipeline trial scored zero. The matching
-control has finished model work but is still awaiting an official result after
-two verifier attempts timed out during dependency installation; the third and
-last permitted attempt is running.
+**Status: running; 2 of 24 trials have finished.** This report is incomplete and
+makes no capability or efficiency improvement claim. The first verification
+candidate pipeline trial scored zero. The matching control also has raw reward
+zero, but its verifier failed during dependency download without starting the
+tests. That raw reward is retained and classified as a verifier dependency error,
+excluded from capability-score comparisons. The operations arm is now running.
 
 The [registered plan](../configs/verification-quality-20261009/experiment.json)
 and [reproduction instructions](../configs/verification-quality-20261009/README.md)
@@ -80,10 +80,30 @@ microbatch count and reverse-order backward processing. This output had no
 successful numerical self-check. The failure is retained without a replacement
 model run.
 
+The control verifier timed out twice at its original 900-second deadline. Its
+third attempt terminated after a network timeout while downloading/extracting
+`nvidia-cufft-cu12`. The unchanged task script then wrote reward zero, even though
+pytest never started. Harbor reported a scored zero with no exception. The
+structured evidence preserves that native status and reward, but records an
+analytical `verifier_error` with no valid capability score. No extra verifier
+attempt or replacement model run is added.
+
 These two trajectories do not show improved verification coverage from the
 expanded verification prompt. They also do not establish a general regression:
 the remaining repetitions and tasks have not finished, dependency-download
-conditions differed, and the control's official result is still pending.
+conditions differed, and the control's raw zero is invalid as capability evidence.
+
+Both operations-arm pipeline agents have also finished; their official verifiers
+are pending. Each still incurred one 120-second timeout by placing `cd ... &&`
+outside the redirected background group. One recovered too late to execute its
+real-model harness and disclosed the limitation. The other passed a single-rank
+comparison of all 39 parameter gradients and completed a two-rank run. Its
+two-rank analysis selected nonzero gradients as owned parameters, which could
+hide an expected gradient that is absent. Its hook harness summed backward
+captures instead of preserving microbatch identity and failed with a shape
+mismatch. The final response disclosed that unresolved activation-check gap.
+These are partial verification results, not evidence that every requirement was
+checked. Neither operations trajectory showed external task-answer exposure.
 
 ## Local and CI validation
 
diff --git a/benchmarks/harbor/results/verification-quality-20261009.json b/benchmarks/harbor/results/verification-quality-20261009.json
index 1e28208..033c45e 100644
--- a/benchmarks/harbor/results/verification-quality-20261009.json
+++ b/benchmarks/harbor/results/verification-quality-20261009.json
@@ -1,6 +1,6 @@
 {
   "schema_version": 1,
-  "generated_at": "2026-10-09T15:37:51.389849+00:00",
+  "generated_at": "2026-10-09T15:59:54.506657+00:00",
   "experiment_plan": "benchmarks/harbor/configs/verification-quality-20261009/experiment.json",
   "experiment_plan_sha256": "b3968a10a29507b72c62f6f886a6af4f982373e15ca6103da21dd288362396c1",
   "status": "running",
@@ -8,7 +8,7 @@
     "control": {
       "planned": 8,
       "statuses": {
-        "verifying": 1,
+        "verifier_error": 1,
         "not_started": 7
       },
       "passes": 0,
@@ -16,7 +16,7 @@
       "known_cost_usd": 0.085583436,
       "cost_complete": false,
       "usage_complete": false,
-      "known_agent_seconds": 0,
+      "known_agent_seconds": 671.625725,
       "model_attempts": 27,
       "tool_timeouts": 1
     },
@@ -38,16 +38,17 @@
     "operations": {
       "planned": 8,
       "statuses": {
-        "not_started": 8
+        "verifying": 2,
+        "not_started": 6
       },
       "passes": 0,
       "scored_failures": 0,
-      "known_cost_usd": 0,
+      "known_cost_usd": 0.135588492,
       "cost_complete": false,
       "usage_complete": false,
       "known_agent_seconds": 0,
-      "model_attempts": 0,
-      "tool_timeouts": 0
+      "model_attempts": 43,
+      "tool_timeouts": 2
     }
   },
   "trials": [
@@ -58,12 +59,49 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
       "job_name": "vq-con-torch-pipeline-parallelism-r1-20261009",
       "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
-      "status": "verifying",
+      "status": "verifier_error",
       "reward": null,
-      "launcher_exit_code": null,
+      "launcher_exit_code": 0,
       "known_cost_usd": 0.085583436,
       "cost_complete": true,
       "usage_complete": true,
+      "raw_verifier_rewards": {
+        "reward": 0.0
+      },
+      "exception_type": null,
+      "result_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__YcctKv5/result.json",
+      "task_checksum": "31e565558cd22598a3c2069f771d9d0eee26fb843b77f7b03e00ad4c336fbce9",
+      "agent_setup_seconds": 137.350091,
+      "agent_execution_seconds": 671.625725,
+      "verifier_seconds": 2345.913464,
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "started_at": "2026-10-09T14:52:06.449794Z",
+      "finished_at": "2026-10-09T15:44:55.652186Z",
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "timeout",
+            "duration_seconds": 900.0030681919961
+          },
+          {
+            "attempt": 2,
+            "status": "timeout",
+            "duration_seconds": 900.0031834670008
+          },
+          {
+            "attempt": 3,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 546.6134462979971
+          }
+        ]
+      },
       "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__YcctKv5/agent/trajectory.json",
       "trajectory_sha256": "698e12648ccb5f0f8312d6ee8565d7cec6185f5aa535ab21db847bcb03b08cc2",
       "terminal": {
@@ -90,7 +128,41 @@
       "recovery_events": 0,
       "tool_timeouts": 1,
       "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
-      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350"
+      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350",
+      "audit": {
+        "job": "vq-con-torch-pipeline-parallelism-r1-20261009",
+        "arm": "control",
+        "task": "terminal-bench/torch-pipeline-parallelism",
+        "repeat": 1,
+        "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__YcctKv5/agent/trajectory.json",
+        "trajectory_sha256": "698e12648ccb5f0f8312d6ee8565d7cec6185f5aa535ab21db847bcb03b08cc2",
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": false,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "partial_pass",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              31,
+              49,
+              51,
+              55
+            ],
+            "note": "The real Llama model passes a single-rank check against its full-batch forward, including every selected microbatch and 39 parameter gradients. The two-rank attempts time out or have no completion output. The final explicitly discloses the two-rank gap. Network calls retrieve general Transformers/PyTorch source and packages."
+          }
+        ],
+        "score_valid_for_capability_comparison": false,
+        "score_classification": "verifier_dependency_error",
+        "official_failure_notes": "The first two verifier attempts timed out at 900 seconds each. Attempt 3 returned normally after uv failed to download/extract nvidia-cufft-cu12 because of a network timeout. The unchanged task script wrote reward 0 despite never starting pytest. Preserve raw reward 0; exclude this trial from capability-score comparisons and count it as a verifier dependency error, not an agent-scored failure.",
+        "exclusion_reason": "verifier_dependency_error"
+      },
+      "native_status": "scored",
+      "native_reward": 0.0,
+      "failure_stage": "verifier_dependencies"
     },
     {
       "arm": "verification",
@@ -158,7 +230,40 @@
       "recovery_events": 0,
       "tool_timeouts": 2,
       "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
-      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c"
+      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c",
+      "audit": {
+        "job": "vq-ver-torch-pipeline-parallelism-r1-20261009",
+        "arm": "verification",
+        "task": "terminal-bench/torch-pipeline-parallelism",
+        "repeat": 1,
+        "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__TxDd3vH/agent/trajectory.json",
+        "trajectory_sha256": "c47b1502cc633647967ba9b7fbd7a99c9528a35180e7e826a6d6bb76b5f938d8",
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": true,
+        "real_entry_check": "not_executed",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              9,
+              11,
+              13,
+              17,
+              19,
+              27,
+              29,
+              37,
+              41
+            ],
+            "note": "Exact-function searches return security challenges, missing endpoints, authentication errors, or no answer links. Two Bash calls time out at 120 seconds. Torch remains unavailable; a real-model harness is written but not executed. Only py_compile passes, and the final explicitly discloses no numerical verification and the chosen loss-scale interpretation."
+          }
+        ],
+        "official_failure_notes": "Completed verifier reward 0: world_size=1 backward activation mismatch at lm_head (max diff 0.5714225); world_size=2 raises AttributeError because local_targets[micro] is None. The candidate chose multiplication by the microbatch count and did not run its numerical harness."
+      }
     },
     {
       "arm": "operations",
@@ -167,12 +272,71 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
       "job_name": "vq-ope-torch-pipeline-parallelism-r1-20261009",
       "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
-      "status": "not_started",
+      "status": "verifying",
       "reward": null,
       "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "known_cost_usd": 0.076849608,
+      "cost_complete": true,
+      "usage_complete": true,
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__bEqs3hp/agent/trajectory.json",
+      "trajectory_sha256": "96d56bbfa98b0e779517f6a4a5c85cfd70d417e2dbe9c318aa343bf814be3807",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 653746.844654,
+        "timestamp": "2026-10-09T15:58:39.067Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "metrics": {
+        "total_steps": 41,
+        "total_prompt_tokens": 711314,
+        "total_completion_tokens": 49473,
+        "total_cached_tokens": 666368,
+        "total_cost_usd": 0.076849608,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.076849608
+        }
+      },
+      "model_attempts": 20,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "recovery_events": 0,
+      "tool_timeouts": 1,
+      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
+      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc",
+      "audit": {
+        "job": "vq-ope-torch-pipeline-parallelism-r1-20261009",
+        "arm": "operations",
+        "task": "terminal-bench/torch-pipeline-parallelism",
+        "repeat": 1,
+        "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__bEqs3hp/agent/trajectory.json",
+        "trajectory_sha256": "96d56bbfa98b0e779517f6a4a5c85cfd70d417e2dbe9c318aa343bf814be3807",
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "not_executed",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "critical_check_coverage": "not_executed",
+        "evidence": [
+          {
+            "steps": [
+              13,
+              21,
+              25,
+              31,
+              33,
+              39,
+              41
+            ],
+            "note": "The agent reads its own bootstrap and live logs, installs packages, and writes a real-model harness without running it. A background AND-list holds captured pipes and times out at 120 seconds. After checking that no installer remains, the agent relaunches with setsid. Only import and partition-helper checks complete; the final explicitly states that end-to-end behavior is unexecuted."
+          }
+        ],
+        "official_failure_notes": "Verifier pending."
+      }
     },
     {
       "arm": "operations",
@@ -181,12 +345,74 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
       "job_name": "vq-ope-torch-pipeline-parallelism-r2-20261009",
       "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
-      "status": "not_started",
+      "status": "verifying",
       "reward": null,
       "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "known_cost_usd": 0.058738884,
+      "cost_complete": true,
+      "usage_complete": true,
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__a3m3xr8/agent/trajectory.json",
+      "trajectory_sha256": "6e4fc530b2124cd82e7f9b41c9b12281ed77eb0a3c49d898494a68f27437fbdd",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 653173.203504,
+        "timestamp": "2026-10-09T15:58:34.005Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "metrics": {
+        "total_steps": 47,
+        "total_prompt_tokens": 724439,
+        "total_completion_tokens": 43079,
+        "total_cached_tokens": 715264,
+        "total_cost_usd": 0.058738884,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.058738884
+        }
+      },
+      "model_attempts": 23,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "recovery_events": 0,
+      "tool_timeouts": 1,
+      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
+      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc",
+      "audit": {
+        "job": "vq-ope-torch-pipeline-parallelism-r2-20261009",
+        "arm": "operations",
+        "task": "terminal-bench/torch-pipeline-parallelism",
+        "repeat": 2,
+        "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__a3m3xr8/agent/trajectory.json",
+        "trajectory_sha256": "6e4fc530b2124cd82e7f9b41c9b12281ed77eb0a3c49d898494a68f27437fbdd",
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "partial_pass",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "critical_check_coverage": "partial",
+        "evidence": [
+          {
+            "steps": [
+              13,
+              19,
+              23,
+              35,
+              37,
+              39,
+              41,
+              43,
+              45,
+              47
+            ],
+            "note": "A real Llama full-batch reference runs. The single-rank comparison covers all 39 parameter gradients and passes at the harness tolerance. The two-rank run completes, but its all-parameter report initially fails on non-owned parameters. The agent then selects nonzero gradients as owned and reports 19/20 passing parameters with matching layer ranges. Excluding non-owned parameters is justified, but selection based on observed nonzero gradients could hide missing expected gradients; it is not an independent ownership assertion. A hook-based per-microbatch check adds backward captures instead of preserving individual items and fails with a 2-versus-6 row shape mismatch. The final discloses this unresolved harness failure and limits runtime verification to CPU/fp32 and one/two ranks. The initial background installer still times out at 120 seconds despite explicit guidance."
+          }
+        ],
+        "official_failure_notes": "Verifier pending."
+      }
     },
     {
       "arm": "verification",
diff --git a/benchmarks/harbor/results/verification-quality-audit-20261009.json b/benchmarks/harbor/results/verification-quality-audit-20261009.json
index 91adb92..56d0583 100644
--- a/benchmarks/harbor/results/verification-quality-audit-20261009.json
+++ b/benchmarks/harbor/results/verification-quality-audit-20261009.json
@@ -11,7 +11,7 @@
       "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__YcctKv5/agent/trajectory.json",
       "trajectory_sha256": "698e12648ccb5f0f8312d6ee8565d7cec6185f5aa535ab21db847bcb03b08cc2",
       "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
-      "eligible_as_independent_capability_evidence": true,
+      "eligible_as_independent_capability_evidence": false,
       "task_specific_search_attempt": false,
       "real_entry_check": "partial_pass",
       "final_limitations_disclosed": true,
@@ -28,7 +28,11 @@
           ],
           "note": "The real Llama model passes a single-rank check against its full-batch forward, including every selected microbatch and 39 parameter gradients. The two-rank attempts time out or have no completion output. The final explicitly discloses the two-rank gap. Network calls retrieve general Transformers/PyTorch source and packages."
         }
-      ]
+      ],
+      "score_valid_for_capability_comparison": false,
+      "score_classification": "verifier_dependency_error",
+      "official_failure_notes": "The first two verifier attempts timed out at 900 seconds each. Attempt 3 returned normally after uv failed to download/extract nvidia-cufft-cu12 because of a network timeout. The unchanged task script wrote reward 0 despite never starting pytest. Preserve raw reward 0; exclude this trial from capability-score comparisons and count it as a verifier dependency error, not an agent-scored failure.",
+      "exclusion_reason": "verifier_dependency_error"
     },
     {
       "job": "vq-ver-torch-pipeline-parallelism-r1-20261009",
@@ -62,6 +66,73 @@
         }
       ],
       "official_failure_notes": "Completed verifier reward 0: world_size=1 backward activation mismatch at lm_head (max diff 0.5714225); world_size=2 raises AttributeError because local_targets[micro] is None. The candidate chose multiplication by the microbatch count and did not run its numerical harness."
+    },
+    {
+      "job": "vq-ope-torch-pipeline-parallelism-r1-20261009",
+      "arm": "operations",
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "repeat": 1,
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__bEqs3hp/agent/trajectory.json",
+      "trajectory_sha256": "96d56bbfa98b0e779517f6a4a5c85cfd70d417e2dbe9c318aa343bf814be3807",
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "not_executed",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "critical_check_coverage": "not_executed",
+      "evidence": [
+        {
+          "steps": [
+            13,
+            21,
+            25,
+            31,
+            33,
+            39,
+            41
+          ],
+          "note": "The agent reads its own bootstrap and live logs, installs packages, and writes a real-model harness without running it. A background AND-list holds captured pipes and times out at 120 seconds. After checking that no installer remains, the agent relaunches with setsid. Only import and partition-helper checks complete; the final explicitly states that end-to-end behavior is unexecuted."
+        }
+      ],
+      "official_failure_notes": "Verifier pending."
+    },
+    {
+      "job": "vq-ope-torch-pipeline-parallelism-r2-20261009",
+      "arm": "operations",
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "repeat": 2,
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__a3m3xr8/agent/trajectory.json",
+      "trajectory_sha256": "6e4fc530b2124cd82e7f9b41c9b12281ed77eb0a3c49d898494a68f27437fbdd",
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "partial_pass",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "critical_check_coverage": "partial",
+      "evidence": [
+        {
+          "steps": [
+            13,
+            19,
+            23,
+            35,
+            37,
+            39,
+            41,
+            43,
+            45,
+            47
+          ],
+          "note": "A real Llama full-batch reference runs. The single-rank comparison covers all 39 parameter gradients and passes at the harness tolerance. The two-rank run completes, but its all-parameter report initially fails on non-owned parameters. The agent then selects nonzero gradients as owned and reports 19/20 passing parameters with matching layer ranges. Excluding non-owned parameters is justified, but selection based on observed nonzero gradients could hide missing expected gradients; it is not an independent ownership assertion. A hook-based per-microbatch check adds backward captures instead of preserving individual items and fails with a 2-versus-6 row shape mismatch. The final discloses this unresolved harness failure and limits runtime verification to CPU/fp32 and one/two ranks. The initial background installer still times out at 120 seconds despite explicit guidance."
+        }
+      ],
+      "official_failure_notes": "Verifier pending."
     }
   ]
 }

From 42833aeb278f31a62254c2df2b2ae4dff77184fc Mon Sep 17 00:00:00 2001
From: minixalpha <xkzalpha@gmail.com>
Date: Fri, 9 Oct 2026 09:42:52 -0700
Subject: [PATCH 6/7] docs(benchmarks): record pipeline comparison outcomes and
 audit gaps

---
 .../reports/verification-quality-20261009.md  |  51 ++-
 .../verification-quality-20261009.json        | 381 ++++++++++++++++--
 .../verification-quality-audit-20261009.json  | 111 ++++-
 3 files changed, 499 insertions(+), 44 deletions(-)

diff --git a/benchmarks/harbor/reports/verification-quality-20261009.md b/benchmarks/harbor/reports/verification-quality-20261009.md
index 3823858..d29e9f4 100644
--- a/benchmarks/harbor/reports/verification-quality-20261009.md
+++ b/benchmarks/harbor/reports/verification-quality-20261009.md
@@ -1,11 +1,16 @@
 # Verification quality and long-command guidance comparison
 
-**Status: running; 2 of 24 trials have finished.** This report is incomplete and
+**Status: running; 6 of 24 trials have finished.** This report is incomplete and
 makes no capability or efficiency improvement claim. The first verification
 candidate pipeline trial scored zero. The matching control also has raw reward
 zero, but its verifier failed during dependency download without starting the
 tests. That raw reward is retained and classified as a verifier dependency error,
-excluded from capability-score comparisons. The operations arm is now running.
+excluded from capability-score comparisons. Both operations pipeline trials also
+scored zero after their tests executed. The second control also encountered a
+verifier dependency error. The second verification candidate passed all four
+official tests. All six pipeline trials have finished. Neither pipeline control
+has a valid capability score, so no fresh paired score comparison is available
+for this task. Scheduler trials are now running.
 
 The [registered plan](../configs/verification-quality-20261009/experiment.json)
 and [reproduction instructions](../configs/verification-quality-20261009/README.md)
@@ -93,8 +98,8 @@ expanded verification prompt. They also do not establish a general regression:
 the remaining repetitions and tasks have not finished, dependency-download
 conditions differed, and the control's raw zero is invalid as capability evidence.
 
-Both operations-arm pipeline agents have also finished; their official verifiers
-are pending. Each still incurred one 120-second timeout by placing `cd ... &&`
+Both operations-arm pipeline trials have also finished. Each still incurred
+one 120-second tool timeout by placing `cd ... &&`
 outside the redirected background group. One recovered too late to execute its
 real-model harness and disclosed the limitation. The other passed a single-rank
 comparison of all 39 parameter gradients and completed a two-rank run. Its
@@ -105,6 +110,44 @@ mismatch. The final response disclosed that unresolved activation-check gap.
 These are partial verification results, not evidence that every requirement was
 checked. Neither operations trajectory showed external task-answer exposure.
 
+Both operations verifiers timed out once during dependency download, then
+completed actual tests on the second attempt with two failures and two passes.
+Both failed single-rank `lm_head.bwd` and two-rank `model.layers.1.bwd`
+microbatch comparisons. The maximum differences were 0.1428563/0.0421933 in
+repeat 1 and 0.0416673/0.0146767 in repeat 2. Successful parameter-gradient
+self-checks in repeat 2 therefore did not establish the required per-microbatch
+activation agreement. No patch-and-rerun causal attribution has been performed.
+
+The second control passed real single-/two-rank checks of per-layer microbatch
+activations, owned parameter gradients, and loss. It fixed test collection/index
+errors while retaining the selected microbatches, and excluded non-owned
+parameters using rank/layer ownership. A check on Transformers 4.38.2 then found
+an implementation compatibility bug. After fixing it, the agent reran 4.38.2 and
+4.46.3; its earlier 4.53.3 result was not refreshed despite the final three-version
+summary. Its official verifier could not fetch the pytest package index and
+never started tests; the native zero is classified as another verifier error.
+
+The second verification candidate ran real checks and fixed broadcast result
+handling, incompatible model inputs, and backward microbatch ordering. It still
+failed two-rank tied embedding/head gradient checks, then changed the harness to
+untied weights and obtained passing numerical flags. That leaves the tied case
+unresolved. Its harness also skips unobserved layers without an independent
+partition assertion. The final model call exhausted the 720-second work budget,
+so there was no final summary and its cost/usage accounting is partial. These
+positive repair actions and remaining evidence gaps are both retained.
+The official verifier subsequently passed all four tests on its first attempt.
+That pass does not resolve the broader tied-weight self-test failure or missing
+final disclosure.
+
+Pipeline interpretation also needs a contract caveat. After all six pipeline
+agent outputs were fixed, the known diagnostic verifier was reviewed again: it
+compares hook records in call order against reference microbatches processed in
+input order. The public AFAB wording does not specify backward microbatch order.
+A self-check that aligns gradients by microbatch identity can therefore validate
+numerical behavior while differing from the verifier's ordering constraint.
+This does not make an incomplete self-check sufficient, and no counterfactual
+official rerun was performed. Hidden tests for the two new tasks remain unread.
+
 ## Local and CI validation
 
 Core tests: **351 passed, 4 skipped**. Harbor workflow tests: **99 passed**.
diff --git a/benchmarks/harbor/results/verification-quality-20261009.json b/benchmarks/harbor/results/verification-quality-20261009.json
index 033c45e..b3f4a16 100644
--- a/benchmarks/harbor/results/verification-quality-20261009.json
+++ b/benchmarks/harbor/results/verification-quality-20261009.json
@@ -1,6 +1,6 @@
 {
   "schema_version": 1,
-  "generated_at": "2026-10-09T15:59:54.506657+00:00",
+  "generated_at": "2026-10-09T16:42:31.828366+00:00",
   "experiment_plan": "benchmarks/harbor/configs/verification-quality-20261009/experiment.json",
   "experiment_plan_sha256": "b3968a10a29507b72c62f6f886a6af4f982373e15ca6103da21dd288362396c1",
   "status": "running",
@@ -8,45 +8,47 @@
     "control": {
       "planned": 8,
       "statuses": {
-        "verifier_error": 1,
-        "not_started": 7
+        "verifier_error": 2,
+        "not_started": 6
       },
       "passes": 0,
       "scored_failures": 0,
-      "known_cost_usd": 0.085583436,
+      "known_cost_usd": 0.155345088,
       "cost_complete": false,
       "usage_complete": false,
-      "known_agent_seconds": 671.625725,
-      "model_attempts": 27,
+      "known_agent_seconds": 1258.6505069999998,
+      "model_attempts": 53,
       "tool_timeouts": 1
     },
     "verification": {
       "planned": 8,
       "statuses": {
-        "scored": 1,
-        "not_started": 7
+        "scored": 2,
+        "started": 1,
+        "not_started": 5
       },
-      "passes": 0,
+      "passes": 1,
       "scored_failures": 1,
-      "known_cost_usd": 0.0612708,
+      "known_cost_usd": 0.164811612,
       "cost_complete": false,
       "usage_complete": false,
-      "known_agent_seconds": 655.750458,
-      "model_attempts": 20,
+      "known_agent_seconds": 1376.596241,
+      "model_attempts": 42,
       "tool_timeouts": 2
     },
     "operations": {
       "planned": 8,
       "statuses": {
-        "verifying": 2,
-        "not_started": 6
+        "scored": 2,
+        "started": 1,
+        "not_started": 5
       },
       "passes": 0,
-      "scored_failures": 0,
+      "scored_failures": 2,
       "known_cost_usd": 0.135588492,
       "cost_complete": false,
       "usage_complete": false,
-      "known_agent_seconds": 0,
+      "known_agent_seconds": 1311.230174,
       "model_attempts": 43,
       "tool_timeouts": 2
     }
@@ -123,6 +125,9 @@
         }
       },
       "model_attempts": 27,
+      "summed_model_call_seconds": 316.959456431,
+      "maximum_model_call_seconds": 73.936952907,
+      "summed_tool_call_seconds": 354.618524057,
       "incomplete_model_attempts": 0,
       "truncated_responses": 0,
       "recovery_events": 0,
@@ -225,6 +230,9 @@
         }
       },
       "model_attempts": 20,
+      "summed_model_call_seconds": 248.364908082,
+      "maximum_model_call_seconds": 61.362616916,
+      "summed_tool_call_seconds": 407.52567982,
       "incomplete_model_attempts": 0,
       "truncated_responses": 0,
       "recovery_events": 0,
@@ -272,12 +280,44 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
       "job_name": "vq-ope-torch-pipeline-parallelism-r1-20261009",
       "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
-      "status": "verifying",
-      "reward": null,
-      "launcher_exit_code": null,
+      "status": "scored",
+      "reward": 0.0,
+      "launcher_exit_code": 0,
       "known_cost_usd": 0.076849608,
       "cost_complete": true,
       "usage_complete": true,
+      "raw_verifier_rewards": {
+        "reward": 0.0
+      },
+      "exception_type": null,
+      "result_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__bEqs3hp/result.json",
+      "task_checksum": "31e565558cd22598a3c2069f771d9d0eee26fb843b77f7b03e00ad4c336fbce9",
+      "agent_setup_seconds": 134.611081,
+      "agent_execution_seconds": 655.875226,
+      "verifier_seconds": 1252.528498,
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "started_at": "2026-10-09T15:45:27.637317Z",
+      "finished_at": "2026-10-09T16:19:45.390521Z",
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "timeout",
+            "duration_seconds": 900.0040100719998
+          },
+          {
+            "attempt": 2,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 351.31837493900093
+          }
+        ]
+      },
       "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__bEqs3hp/agent/trajectory.json",
       "trajectory_sha256": "96d56bbfa98b0e779517f6a4a5c85cfd70d417e2dbe9c318aa343bf814be3807",
       "terminal": {
@@ -299,6 +339,9 @@
         }
       },
       "model_attempts": 20,
+      "summed_model_call_seconds": 243.921774978,
+      "maximum_model_call_seconds": 167.232692041,
+      "summed_tool_call_seconds": 409.771602606,
       "incomplete_model_attempts": 0,
       "truncated_responses": 0,
       "recovery_events": 0,
@@ -335,7 +378,11 @@
             "note": "The agent reads its own bootstrap and live logs, installs packages, and writes a real-model harness without running it. A background AND-list holds captured pipes and times out at 120 seconds. After checking that no installer remains, the agent relaunches with setsid. Only import and partition-helper checks complete; the final explicitly states that end-to-end behavior is unexecuted."
           }
         ],
-        "official_failure_notes": "Verifier pending."
+        "official_failure_notes": "After one dependency-download timeout, the second verifier attempt executes pytest and completes with reward 0 (2 failed, 2 passed). Single-rank lm_head.bwd microbatch 0 max difference: 0.1428563446; two-rank model.layers.1.bwd microbatch 0 max difference: 0.0421932563. These are genuine numerical test failures, unlike the first control verifier dependency error.",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_failure",
+        "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__bEqs3hp/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "b5b8a9d7d518a129335b76707c8d9fb1dbea7172228ac9bd9bc4251eeb7e28c5"
       }
     },
     {
@@ -345,12 +392,44 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
       "job_name": "vq-ope-torch-pipeline-parallelism-r2-20261009",
       "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
-      "status": "verifying",
-      "reward": null,
-      "launcher_exit_code": null,
+      "status": "scored",
+      "reward": 0.0,
+      "launcher_exit_code": 0,
       "known_cost_usd": 0.058738884,
       "cost_complete": true,
       "usage_complete": true,
+      "raw_verifier_rewards": {
+        "reward": 0.0
+      },
+      "exception_type": null,
+      "result_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__a3m3xr8/result.json",
+      "task_checksum": "31e565558cd22598a3c2069f771d9d0eee26fb843b77f7b03e00ad4c336fbce9",
+      "agent_setup_seconds": 138.896958,
+      "agent_execution_seconds": 655.354948,
+      "verifier_seconds": 1244.67741,
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "started_at": "2026-10-09T15:45:18.428047Z",
+      "finished_at": "2026-10-09T16:19:32.758980Z",
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "timeout",
+            "duration_seconds": 900.0031564370001
+          },
+          {
+            "attempt": 2,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 343.4773280070003
+          }
+        ]
+      },
       "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__a3m3xr8/agent/trajectory.json",
       "trajectory_sha256": "6e4fc530b2124cd82e7f9b41c9b12281ed77eb0a3c49d898494a68f27437fbdd",
       "terminal": {
@@ -372,6 +451,9 @@
         }
       },
       "model_attempts": 23,
+      "summed_model_call_seconds": 235.563146319,
+      "maximum_model_call_seconds": 106.58585946199999,
+      "summed_tool_call_seconds": 417.548233131,
       "incomplete_model_attempts": 0,
       "truncated_responses": 0,
       "recovery_events": 0,
@@ -411,7 +493,11 @@
             "note": "A real Llama full-batch reference runs. The single-rank comparison covers all 39 parameter gradients and passes at the harness tolerance. The two-rank run completes, but its all-parameter report initially fails on non-owned parameters. The agent then selects nonzero gradients as owned and reports 19/20 passing parameters with matching layer ranges. Excluding non-owned parameters is justified, but selection based on observed nonzero gradients could hide missing expected gradients; it is not an independent ownership assertion. A hook-based per-microbatch check adds backward captures instead of preserving individual items and fails with a 2-versus-6 row shape mismatch. The final discloses this unresolved harness failure and limits runtime verification to CPU/fp32 and one/two ranks. The initial background installer still times out at 120 seconds despite explicit guidance."
           }
         ],
-        "official_failure_notes": "Verifier pending."
+        "official_failure_notes": "After one dependency-download timeout, the second verifier attempt executes pytest and completes with reward 0 (2 failed, 2 passed). Single-rank lm_head.bwd microbatch 0 max difference: 0.0416672938; two-rank model.layers.1.bwd microbatch 0 max difference: 0.0146767143. These are genuine numerical test failures, unlike the first control verifier dependency error.",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_failure",
+        "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__a3m3xr8/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "54ed7253a7377c20b895455196ab02ed8077fcd34a7465d6f9e99485b6dc5396"
       }
     },
     {
@@ -421,12 +507,115 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
       "job_name": "vq-ver-torch-pipeline-parallelism-r2-20261009",
       "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
+      "status": "scored",
+      "reward": 1.0,
+      "launcher_exit_code": 0,
+      "known_cost_usd": 0.103540812,
       "cost_complete": false,
-      "usage_complete": false
+      "usage_complete": false,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "result_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__TUMs5AX/result.json",
+      "task_checksum": "31e565558cd22598a3c2069f771d9d0eee26fb843b77f7b03e00ad4c336fbce9",
+      "agent_setup_seconds": 104.088802,
+      "agent_execution_seconds": 720.845783,
+      "verifier_seconds": 463.852849,
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "started_at": "2026-10-09T16:19:57.988835Z",
+      "finished_at": "2026-10-09T16:41:41.882654Z",
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 463.49382196100487
+          }
+        ]
+      },
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__TUMs5AX/agent/trajectory.json",
+      "trajectory_sha256": "db9d1cb521b78bcdd040b07393c605f8bbbcdd0bf3f299d7c52eaf34a2563a2b",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 720014.652115,
+        "timestamp": "2026-10-09T16:33:43.843Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "time_budget_exhausted"
+      },
+      "metrics": {
+        "total_steps": 45,
+        "extra": {
+          "usage_complete": false,
+          "known_cost_usd": 0.103540812,
+          "cost_is_partial": true,
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.103540812
+        }
+      },
+      "model_attempts": 22,
+      "summed_model_call_seconds": 493.542580079,
+      "maximum_model_call_seconds": 237.944808967,
+      "summed_tool_call_seconds": 226.41692674,
+      "incomplete_model_attempts": 1,
+      "truncated_responses": 0,
+      "recovery_events": 0,
+      "tool_timeouts": 0,
+      "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
+      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c",
+      "audit": {
+        "job": "vq-ver-torch-pipeline-parallelism-r2-20261009",
+        "arm": "verification",
+        "task": "terminal-bench/torch-pipeline-parallelism",
+        "repeat": 2,
+        "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__TUMs5AX/agent/trajectory.json",
+        "trajectory_sha256": "db9d1cb521b78bcdd040b07393c605f8bbbcdd0bf3f299d7c52eaf34a2563a2b",
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "partial_pass",
+        "final_limitations_disclosed": false,
+        "unsupported_completion_claim_observed": false,
+        "unsupported_intermediate_claim_observed": true,
+        "unresolved_failed_check_observed": true,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "critical_check_coverage": "partial",
+        "evidence": [
+          {
+            "steps": [
+              29,
+              31,
+              33,
+              35,
+              37,
+              39,
+              41
+            ],
+            "note": "A six-layer real Llama harness compares per-microbatch forward/backward activations and selected parameter gradients against a full-batch reference. It exposes and prompts fixes for broadcast_object_list result handling and mutually exclusive input_ids/inputs_embeds. Backward hook captures are changed from overwrites to lists; the implementation then changes backward processing to forward microbatch order, eliminating observed per-item mismatches. This is real defect-finding and repair evidence."
+          },
+          {
+            "steps": [
+              39,
+              41,
+              43,
+              45
+            ],
+            "note": "The intermediate statement that parameter gradients match conflicts with observed embedding/head gradient failures. Two-rank tied-weight parameter checks remain failing after the ordering fix. The next test changes tie_word_embeddings from True to False, and both ranks report success; the tied case is not repaired or rechecked. The harness skips unobserved layers, does not independently assert the layer partition, and labels its aggregate numerical flag layers_per_rank_ok. It also does not check the final norm gradient despite a broader comment. The final model call exhausts the wall-clock budget and produces no final summary or disclosure. Its cost and token accounting are incomplete."
+          }
+        ],
+        "official_failure_notes": "The first verifier attempt completes actual pytest execution with reward 1: all four tests pass. This official pass does not repair the unreported tied-weight self-test failure or make incomplete accounting complete.",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__TUMs5AX/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "931c5d89033bae35cf65965dd76519f77eb2b899ac50ebbfc96564fd755e4e73"
+      }
     },
     {
       "arm": "control",
@@ -435,12 +624,124 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
       "job_name": "vq-con-torch-pipeline-parallelism-r2-20261009",
       "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
-      "status": "not_started",
+      "status": "verifier_error",
       "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "launcher_exit_code": 0,
+      "known_cost_usd": 0.069761652,
+      "cost_complete": true,
+      "usage_complete": true,
+      "raw_verifier_rewards": {
+        "reward": 0.0
+      },
+      "exception_type": null,
+      "result_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__nuNNDUP/result.json",
+      "task_checksum": "31e565558cd22598a3c2069f771d9d0eee26fb843b77f7b03e00ad4c336fbce9",
+      "agent_setup_seconds": 100.827538,
+      "agent_execution_seconds": 587.024782,
+      "verifier_seconds": 136.906223,
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "started_at": "2026-10-09T16:19:57.731639Z",
+      "finished_at": "2026-10-09T16:33:56.359576Z",
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 136.53335461999814
+          }
+        ]
+      },
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__nuNNDUP/agent/trajectory.json",
+      "trajectory_sha256": "c523402ba377e835015d602745083c12d3f8de5a6238ad760072257bad973634",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 586163.607139,
+        "timestamp": "2026-10-09T16:31:26.693Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "metrics": {
+        "total_steps": 53,
+        "total_prompt_tokens": 978643,
+        "total_completion_tokens": 47545,
+        "total_cached_tokens": 955392,
+        "total_cost_usd": 0.069761652,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.069761652
+        }
+      },
+      "model_attempts": 26,
+      "summed_model_call_seconds": 242.30997215399998,
+      "maximum_model_call_seconds": 129.647188506,
+      "summed_tool_call_seconds": 343.783995908,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "recovery_events": 0,
+      "tool_timeouts": 0,
+      "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
+      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350",
+      "audit": {
+        "job": "vq-con-torch-pipeline-parallelism-r2-20261009",
+        "arm": "control",
+        "task": "terminal-bench/torch-pipeline-parallelism",
+        "repeat": 2,
+        "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__nuNNDUP/agent/trajectory.json",
+        "trajectory_sha256": "c523402ba377e835015d602745083c12d3f8de5a6238ad760072257bad973634",
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": false,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": true,
+        "critical_check_coverage": "partial",
+        "evidence": [
+          {
+            "steps": [
+              23,
+              25,
+              27,
+              29,
+              31,
+              33,
+              35,
+              37,
+              39
+            ],
+            "note": "A small actual Llama full-batch forward provides the reference. Initial hooks overwrite microbatch records; the agent changes them to lists, fixes reference slicing, and aligns backward records with reverse microbatch order while retaining every selected microbatch. Parameter checks exclude non-owned parameters using layer/rank ownership, rather than observed gradient values. Both ranks pass per-layer activations, gradients, owned parameter gradients, and loss comparisons. There is no successful separate lm_head activation comparison in this harness."
+          },
+          {
+            "steps": [
+              41,
+              43,
+              45,
+              47,
+              49,
+              51,
+              53
+            ],
+            "note": "Checks pass on Transformers 4.46.3 and 4.53.3; 4.38.2 exposes an unsupported past_key_values keyword. The source is fixed at step 47 and retested on 4.38.2 and 4.46.3. Version 4.53.3 is not rerun after that change, but the final says verification is complete across all three versions. This is a stale final-version claim, not a claim that the entire implementation was untested. CPU/fp32 and model-size limitations are disclosed."
+          }
+        ],
+        "official_failure_notes": "The first verifier attempt returned normally after uv failed to fetch the pytest package index following request retries. Pytest never started. The unchanged script wrote reward 0 and Harbor reported no exception. Classify as a verifier dependency error; preserve native reward 0 and do not retry a non-timeout result.",
+        "score_valid_for_capability_comparison": false,
+        "score_classification": "verifier_dependency_error",
+        "exclusion_reason": "verifier_dependency_error",
+        "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__nuNNDUP/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "6b1a54415cdc40d7b5a80e6a81e00bc6a469d1de80e90bf2e229b1e3528f7c87",
+        "final_claim_caveat": "The three-version summary includes a 4.53.3 check performed before the final implementation edit; mark that evidence stale rather than treating the entire completion statement as unsupported."
+      },
+      "native_status": "scored",
+      "native_reward": 0.0,
+      "failure_stage": "verifier_dependencies"
     },
     {
       "arm": "verification",
@@ -449,12 +750,14 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
       "job_name": "vq-ver-llm-inference-batching-scheduler-r1-20261009",
       "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
-      "status": "not_started",
+      "status": "started",
       "reward": null,
       "launcher_exit_code": null,
       "known_cost_usd": null,
       "cost_complete": false,
-      "usage_complete": false
+      "usage_complete": false,
+      "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
+      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c"
     },
     {
       "arm": "operations",
@@ -463,12 +766,14 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
       "job_name": "vq-ope-llm-inference-batching-scheduler-r1-20261009",
       "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
-      "status": "not_started",
+      "status": "started",
       "reward": null,
       "launcher_exit_code": null,
       "known_cost_usd": null,
       "cost_complete": false,
-      "usage_complete": false
+      "usage_complete": false,
+      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
+      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc"
     },
     {
       "arm": "control",
diff --git a/benchmarks/harbor/results/verification-quality-audit-20261009.json b/benchmarks/harbor/results/verification-quality-audit-20261009.json
index 56d0583..28f6508 100644
--- a/benchmarks/harbor/results/verification-quality-audit-20261009.json
+++ b/benchmarks/harbor/results/verification-quality-audit-20261009.json
@@ -97,7 +97,11 @@
           "note": "The agent reads its own bootstrap and live logs, installs packages, and writes a real-model harness without running it. A background AND-list holds captured pipes and times out at 120 seconds. After checking that no installer remains, the agent relaunches with setsid. Only import and partition-helper checks complete; the final explicitly states that end-to-end behavior is unexecuted."
         }
       ],
-      "official_failure_notes": "Verifier pending."
+      "official_failure_notes": "After one dependency-download timeout, the second verifier attempt executes pytest and completes with reward 0 (2 failed, 2 passed). Single-rank lm_head.bwd microbatch 0 max difference: 0.1428563446; two-rank model.layers.1.bwd microbatch 0 max difference: 0.0421932563. These are genuine numerical test failures, unlike the first control verifier dependency error.",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_failure",
+      "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__bEqs3hp/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "b5b8a9d7d518a129335b76707c8d9fb1dbea7172228ac9bd9bc4251eeb7e28c5"
     },
     {
       "job": "vq-ope-torch-pipeline-parallelism-r2-20261009",
@@ -132,7 +136,110 @@
           "note": "A real Llama full-batch reference runs. The single-rank comparison covers all 39 parameter gradients and passes at the harness tolerance. The two-rank run completes, but its all-parameter report initially fails on non-owned parameters. The agent then selects nonzero gradients as owned and reports 19/20 passing parameters with matching layer ranges. Excluding non-owned parameters is justified, but selection based on observed nonzero gradients could hide missing expected gradients; it is not an independent ownership assertion. A hook-based per-microbatch check adds backward captures instead of preserving individual items and fails with a 2-versus-6 row shape mismatch. The final discloses this unresolved harness failure and limits runtime verification to CPU/fp32 and one/two ranks. The initial background installer still times out at 120 seconds despite explicit guidance."
         }
       ],
-      "official_failure_notes": "Verifier pending."
+      "official_failure_notes": "After one dependency-download timeout, the second verifier attempt executes pytest and completes with reward 0 (2 failed, 2 passed). Single-rank lm_head.bwd microbatch 0 max difference: 0.0416672938; two-rank model.layers.1.bwd microbatch 0 max difference: 0.0146767143. These are genuine numerical test failures, unlike the first control verifier dependency error.",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_failure",
+      "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__a3m3xr8/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "54ed7253a7377c20b895455196ab02ed8077fcd34a7465d6f9e99485b6dc5396"
+    },
+    {
+      "job": "vq-con-torch-pipeline-parallelism-r2-20261009",
+      "arm": "control",
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "repeat": 2,
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__nuNNDUP/agent/trajectory.json",
+      "trajectory_sha256": "c523402ba377e835015d602745083c12d3f8de5a6238ad760072257bad973634",
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": false,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": true,
+      "critical_check_coverage": "partial",
+      "evidence": [
+        {
+          "steps": [
+            23,
+            25,
+            27,
+            29,
+            31,
+            33,
+            35,
+            37,
+            39
+          ],
+          "note": "A small actual Llama full-batch forward provides the reference. Initial hooks overwrite microbatch records; the agent changes them to lists, fixes reference slicing, and aligns backward records with reverse microbatch order while retaining every selected microbatch. Parameter checks exclude non-owned parameters using layer/rank ownership, rather than observed gradient values. Both ranks pass per-layer activations, gradients, owned parameter gradients, and loss comparisons. There is no successful separate lm_head activation comparison in this harness."
+        },
+        {
+          "steps": [
+            41,
+            43,
+            45,
+            47,
+            49,
+            51,
+            53
+          ],
+          "note": "Checks pass on Transformers 4.46.3 and 4.53.3; 4.38.2 exposes an unsupported past_key_values keyword. The source is fixed at step 47 and retested on 4.38.2 and 4.46.3. Version 4.53.3 is not rerun after that change, but the final says verification is complete across all three versions. This is a stale final-version claim, not a claim that the entire implementation was untested. CPU/fp32 and model-size limitations are disclosed."
+        }
+      ],
+      "official_failure_notes": "The first verifier attempt returned normally after uv failed to fetch the pytest package index following request retries. Pytest never started. The unchanged script wrote reward 0 and Harbor reported no exception. Classify as a verifier dependency error; preserve native reward 0 and do not retry a non-timeout result.",
+      "score_valid_for_capability_comparison": false,
+      "score_classification": "verifier_dependency_error",
+      "exclusion_reason": "verifier_dependency_error",
+      "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__nuNNDUP/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "6b1a54415cdc40d7b5a80e6a81e00bc6a469d1de80e90bf2e229b1e3528f7c87",
+      "final_claim_caveat": "The three-version summary includes a 4.53.3 check performed before the final implementation edit; mark that evidence stale rather than treating the entire completion statement as unsupported."
+    },
+    {
+      "job": "vq-ver-torch-pipeline-parallelism-r2-20261009",
+      "arm": "verification",
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "repeat": 2,
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__TUMs5AX/agent/trajectory.json",
+      "trajectory_sha256": "db9d1cb521b78bcdd040b07393c605f8bbbcdd0bf3f299d7c52eaf34a2563a2b",
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "partial_pass",
+      "final_limitations_disclosed": false,
+      "unsupported_completion_claim_observed": false,
+      "unsupported_intermediate_claim_observed": true,
+      "unresolved_failed_check_observed": true,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "critical_check_coverage": "partial",
+      "evidence": [
+        {
+          "steps": [
+            29,
+            31,
+            33,
+            35,
+            37,
+            39,
+            41
+          ],
+          "note": "A six-layer real Llama harness compares per-microbatch forward/backward activations and selected parameter gradients against a full-batch reference. It exposes and prompts fixes for broadcast_object_list result handling and mutually exclusive input_ids/inputs_embeds. Backward hook captures are changed from overwrites to lists; the implementation then changes backward processing to forward microbatch order, eliminating observed per-item mismatches. This is real defect-finding and repair evidence."
+        },
+        {
+          "steps": [
+            39,
+            41,
+            43,
+            45
+          ],
+          "note": "The intermediate statement that parameter gradients match conflicts with observed embedding/head gradient failures. Two-rank tied-weight parameter checks remain failing after the ordering fix. The next test changes tie_word_embeddings from True to False, and both ranks report success; the tied case is not repaired or rechecked. The harness skips unobserved layers, does not independently assert the layer partition, and labels its aggregate numerical flag layers_per_rank_ok. It also does not check the final norm gradient despite a broader comment. The final model call exhausts the wall-clock budget and produces no final summary or disclosure. Its cost and token accounting are incomplete."
+        }
+      ],
+      "official_failure_notes": "The first verifier attempt completes actual pytest execution with reward 1: all four tests pass. This official pass does not repair the unreported tied-weight self-test failure or make incomplete accounting complete.",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__TUMs5AX/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "931c5d89033bae35cf65965dd76519f77eb2b899ac50ebbfc96564fd755e4e73"
     }
   ]
 }

From 030ef7b5632f8a9eab796e00c874bda0a4e8acbf Mon Sep 17 00:00:00 2001
From: minixalpha <xkzalpha@gmail.com>
Date: Sun, 11 Oct 2026 07:10:04 -0700
Subject: [PATCH 7/7] test(benchmarks): finalize verification quality
 comparison evidence

---
 .../verification-quality-20261009/README.md   |   24 +
 .../reports/verification-quality-20261009.md  |  436 ++-
 .../verification-quality-20261009.json        | 2659 ++++++++++++++---
 .../verification-quality-audit-20261009.json  |  766 ++++-
 ...rification-quality-integrity-20261009.json |   72 +
 .../harbor/scripts/collect_comparison.py      |  201 ++
 .../harbor/tests/test_comparison_results.py   |   67 +
 7 files changed, 3669 insertions(+), 556 deletions(-)
 create mode 100644 benchmarks/harbor/results/verification-quality-integrity-20261009.json
 create mode 100644 benchmarks/harbor/scripts/collect_comparison.py
 create mode 100644 benchmarks/harbor/tests/test_comparison_results.py

diff --git a/benchmarks/harbor/configs/verification-quality-20261009/README.md b/benchmarks/harbor/configs/verification-quality-20261009/README.md
index c55af8e..1213eb5 100644
--- a/benchmarks/harbor/configs/verification-quality-20261009/README.md
+++ b/benchmarks/harbor/configs/verification-quality-20261009/README.md
@@ -102,3 +102,27 @@ timeouts, truncation/recovery, estimated cost, and accounting completeness.
 Two repeats and two tasks outside prior tuning are insufficient to establish a
 general capability gain. Report every regression and cost increase. Do not
 promote a candidate based solely on pipeline or one favorable trial.
+
+## Completed results and read-only collection
+
+All 24 model trials finished on October 9, 2026. See the
+[final report](../../reports/verification-quality-20261009.md). Both pipeline
+controls have native reward zero but verifier dependency failures before tests;
+they are excluded from capability comparisons without replacing the trials.
+
+With the original local job artifacts and frozen worktrees still present,
+regenerate results from the completed manual audit without invoking any model
+or verifier:
+
+```bash
+python benchmarks/harbor/scripts/collect_comparison.py \
+  --plan benchmarks/harbor/configs/verification-quality-20261009/experiment.json \
+  --audit benchmarks/harbor/results/verification-quality-audit-20261009.json \
+  --output benchmarks/harbor/results/verification-quality-20261009.json
+```
+
+The collector checks registration, task/config/prompt/source hashes, installed
+wheels, reviewed evidence hashes and retry limits before writing results. The
+audit remains a manual judgment with cited trajectory steps; the collector does
+not infer verification quality from a reward or shell exit status. Missing cost
+is unknown, and partial accounting is retained as a known lower bound.
diff --git a/benchmarks/harbor/reports/verification-quality-20261009.md b/benchmarks/harbor/reports/verification-quality-20261009.md
index d29e9f4..da8ed75 100644
--- a/benchmarks/harbor/reports/verification-quality-20261009.md
+++ b/benchmarks/harbor/reports/verification-quality-20261009.md
@@ -1,157 +1,285 @@
 # Verification quality and long-command guidance comparison
 
-**Status: running; 6 of 24 trials have finished.** This report is incomplete and
-makes no capability or efficiency improvement claim. The first verification
-candidate pipeline trial scored zero. The matching control also has raw reward
-zero, but its verifier failed during dependency download without starting the
-tests. That raw reward is retained and classified as a verifier dependency error,
-excluded from capability-score comparisons. Both operations pipeline trials also
-scored zero after their tests executed. The second control also encountered a
-verifier dependency error. The second verification candidate passed all four
-official tests. All six pipeline trials have finished. Neither pipeline control
-has a valid capability score, so no fresh paired score comparison is available
-for this task. Scheduler trials are now running.
-
-The [registered plan](../configs/verification-quality-20261009/experiment.json)
-and [reproduction instructions](../configs/verification-quality-20261009/README.md)
-freeze three variants, four task revisions, and two repeats per task and arm.
-The [structured results](../results/verification-quality-20261009.json) retain
-pending outcomes separately from completed scores. The [trajectory audit](../results/verification-quality-audit-20261009.json)
-is manual and unblinded; its current entries cover only completed agent runs.
-
-## Frozen variants
-
-| Arm | Source | Prompt words | Change |
+**Completed: 24/24 trials, ending 2026-10-09 18:09:40 UTC.** On the six
+common eligible task/repeat cases, control passes **5/6**, verification **6/6**,
+and operations **6/6**. Both candidates gain only the first scheduler repeat;
+neither regresses on these six cases. All arms pass **4/4** cases on the two
+tasks outside prior tuning. This is an exploratory signal, not evidence of a
+stable general improvement. Keep the candidate experimental; do not promote it
+from these results alone.
+
+The verification prompt produces more observed checks that reject deliberately
+incorrect results, but still abandons a failing tied-weight pipeline case. The
+operations prompt does not eliminate tool timeouts. These findings favor
+improving requirement coverage and evidence tracking, rather than treating a
+longer prompt or a passing official score as sufficient verification.
+
+## Evidence and frozen design
+
+The [registered plan](../configs/verification-quality-20261009/experiment.json),
+[reproduction instructions](../configs/verification-quality-20261009/README.md),
+[structured results](../results/verification-quality-20261009.json),
+[24-trial audit](../results/verification-quality-audit-20261009.json), and
+[integrity record](../results/verification-quality-integrity-20261009.json)
+retain all outcomes, artifact hashes and evidence references. The audit is
+manual and unblinded. Local job directories contain the full private traces;
+published results contain metadata and reviewed evidence summaries.
+
+This follows the [last pilot20 comparison](headless-p0-guarded-20261007.md), whose
+candidate scored 19/20 with pipeline parallelism as its only scored failure.
+That earlier result belongs to its frozen 368-word prompt, not the later
+385-word reference clarification used as this experiment's control. The earlier
+common eligible comparison excludes MTEB due to reference/test exposure. This
+four-task experiment is **not a new twenty-task score**.
+
+| Arm | Frozen runtime source | Prompt words | Change |
 | --- | --- | ---: | --- |
-| Control | `2eb99b608dfb39464d17d109a4a8f11afda607a9` | 385 | Main with the task-supplied-reference clarification |
-| Verification | `3b69d6d` | 482 | Concrete evidence selection, real-entry checks, item coverage, justified test corrections, and final-version evidence review |
-| Operations | `ca624b0` | 480 | Explicit Bash limits, redirected background commands, completion status, and intermediate deliverables |
-
-Only `HEADLESS_SYSTEM_PROMPT` differs in the frozen runtime sources. Removing
-that assignment produces identical agent ASTs; all other runtime/adapter Python
-files are unchanged. The primary branch also contains experiment orchestration,
-but every trial is built by the standard runner from its isolated frozen source.
-
-All arms use the official DeepSeek endpoint with `deepseek/deepseek-flash`,
-100 replies, and 65,536 output tokens per response. Temperature and reasoning
-effort are not explicitly configured. At most two trials run concurrently.
-Whole-trial retries are disabled. Only verifier timeouts retry, at most three
-attempts with the same agent output and original per-attempt deadline.
-
-## Scope and setup
-
-The diagnostic tasks are `torch-pipeline-parallelism` (720 seconds of model
-work) and `llm-inference-batching-scheduler` (1,620 seconds). The two tasks outside
-the previous pilot20 tuning cohort are `build-cython-ext` and
-`cancel-async-tasks` (720 seconds each). Each task runs twice in each arm.
-This is an exploratory subset comparison, not a new twenty-task score.
-
-The experiment designer previously inspected pipeline evaluation code during
-post hoc diagnosis. Neither candidate receives a task-specific backward-order
-rule, hidden expected answer, function name, or model configuration. Candidate
-text was drafted before reading the new tasks' public instructions. Their hidden
-tests and solutions remain uninspected until every scheduled output is fixed.
-
-[Install-only evidence](../results/verification-quality-preflight-20261009.json)
-records four successful control task installations and one successful wheel
-installation for each candidate. All four verifier-dependency profiles are
-explicitly `not_configured`; these installation checks do not certify official
-verifier dependencies. The original task image IDs are recorded, and both
-diagnostic images match the prior pilot. Task files and official scripts remain
-unchanged.
-
-## Initial observations, pending the remaining trials
-
-The first control pipeline trajectory exercised a small real Llama model. Its
-single-rank check compared all selected microbatch activations and 39 parameter
-gradients against a full-batch reference through the model's own forward path.
-That check passed. Two-rank checks timed out or produced no completion output;
-the final response disclosed this gap. Its network requests retrieved general
-Transformers/PyTorch source and packages, with no task-answer retrieval observed.
-
-The first verification-candidate pipeline trajectory saved an implementation and
-passed `py_compile`, but could not import Torch. It wrote a numerical harness
-without executing it. Two Bash commands hit the 120-second tool limit, including
-one with an inner `timeout 300`. The final response explicitly disclosed that
-runtime behavior was unverified. Exact-function searches returned authentication
-errors, missing endpoints, security challenges, or no answer links; no task-answer
-content was observed. These attempts are recorded even without confirmed exposure.
-
-The candidate's official verifier completed with reward zero: the single-rank
-case failed a backward-activation comparison (`lm_head.bwd`, maximum difference
-approximately 0.5714225), while the two-rank case raised `AttributeError` because
-`local_targets[micro]` was `None`. The implementation chose multiplication by the
-microbatch count and reverse-order backward processing. This output had no
-successful numerical self-check. The failure is retained without a replacement
-model run.
-
-The control verifier timed out twice at its original 900-second deadline. Its
-third attempt terminated after a network timeout while downloading/extracting
-`nvidia-cufft-cu12`. The unchanged task script then wrote reward zero, even though
-pytest never started. Harbor reported a scored zero with no exception. The
-structured evidence preserves that native status and reward, but records an
-analytical `verifier_error` with no valid capability score. No extra verifier
-attempt or replacement model run is added.
-
-These two trajectories do not show improved verification coverage from the
-expanded verification prompt. They also do not establish a general regression:
-the remaining repetitions and tasks have not finished, dependency-download
-conditions differed, and the control's raw zero is invalid as capability evidence.
-
-Both operations-arm pipeline trials have also finished. Each still incurred
-one 120-second tool timeout by placing `cd ... &&`
-outside the redirected background group. One recovered too late to execute its
-real-model harness and disclosed the limitation. The other passed a single-rank
-comparison of all 39 parameter gradients and completed a two-rank run. Its
-two-rank analysis selected nonzero gradients as owned parameters, which could
-hide an expected gradient that is absent. Its hook harness summed backward
-captures instead of preserving microbatch identity and failed with a shape
-mismatch. The final response disclosed that unresolved activation-check gap.
-These are partial verification results, not evidence that every requirement was
-checked. Neither operations trajectory showed external task-answer exposure.
-
-Both operations verifiers timed out once during dependency download, then
-completed actual tests on the second attempt with two failures and two passes.
-Both failed single-rank `lm_head.bwd` and two-rank `model.layers.1.bwd`
-microbatch comparisons. The maximum differences were 0.1428563/0.0421933 in
-repeat 1 and 0.0416673/0.0146767 in repeat 2. Successful parameter-gradient
-self-checks in repeat 2 therefore did not establish the required per-microbatch
-activation agreement. No patch-and-rerun causal attribution has been performed.
-
-The second control passed real single-/two-rank checks of per-layer microbatch
-activations, owned parameter gradients, and loss. It fixed test collection/index
-errors while retaining the selected microbatches, and excluded non-owned
-parameters using rank/layer ownership. A check on Transformers 4.38.2 then found
-an implementation compatibility bug. After fixing it, the agent reran 4.38.2 and
-4.46.3; its earlier 4.53.3 result was not refreshed despite the final three-version
-summary. Its official verifier could not fetch the pytest package index and
-never started tests; the native zero is classified as another verifier error.
-
-The second verification candidate ran real checks and fixed broadcast result
-handling, incompatible model inputs, and backward microbatch ordering. It still
-failed two-rank tied embedding/head gradient checks, then changed the harness to
-untied weights and obtained passing numerical flags. That leaves the tied case
-unresolved. Its harness also skips unobserved layers without an independent
-partition assertion. The final model call exhausted the 720-second work budget,
-so there was no final summary and its cost/usage accounting is partial. These
-positive repair actions and remaining evidence gaps are both retained.
-The official verifier subsequently passed all four tests on its first attempt.
-That pass does not resolve the broader tied-weight self-test failure or missing
-final disclosure.
-
-Pipeline interpretation also needs a contract caveat. After all six pipeline
-agent outputs were fixed, the known diagnostic verifier was reviewed again: it
-compares hook records in call order against reference microbatches processed in
-input order. The public AFAB wording does not specify backward microbatch order.
-A self-check that aligns gradients by microbatch identity can therefore validate
-numerical behavior while differing from the verifier's ordering constraint.
-This does not make an incomplete self-check sufficient, and no counterfactual
-official rerun was performed. Hidden tests for the two new tasks remain unread.
-
-## Local and CI validation
-
-Core tests: **351 passed, 4 skipped**. Harbor workflow tests: **99 passed**.
-The eight orchestration tests passed again after tightening the preflight gate
-to require every planned installation result. Draft PR #48 passed the Python
-3.13, Python 3.14, and Harbor workflow CI checks. Research notes are unchanged in
-both languages.
+| Control | `2eb99b608dfb39464d17d109a4a8f11afda607a9` | 385 | Main with task-supplied-reference clarification |
+| Verification | `3b69d6d54cec29bbeaa129914419c90a70fb87d7` | 482 | Real-entry checks, item coverage, justified test corrections, negative controls, final-version evidence |
+| Operations | `ca624b0c02c5c543dffd3c99efbad031d7658afb` | 480 | Bash deadlines, background redirection, completion status, intermediate deliverables |
+
+Only `HEADLESS_SYSTEM_PROMPT` differs between frozen runtimes. The agent AST
+without that assignment is identical; other runtime/adapter Python files are
+identical, apart from generated package-version metadata. No finish gate or
+runtime verification machinery was added. Each candidate is a bundle of prompt
+changes; individual sentences cannot be credited separately.
+
+Each arm runs four tasks twice. Pipeline, Cython and async work budgets are
+720 seconds; scheduler is 1,620 seconds. All use `deepseek/deepseek-flash` at the
+official DeepSeek endpoint, 100 replies and 65,536 output tokens per request.
+Temperature and reasoning effort are unset. Setup has a separate 1,800-second
+limit. The queue rotates arms, runs at most two trials concurrently, and waits
+for each pair to finish. Repeat indices are not shared random seeds.
+
+There are no replacement model runs or whole-trial retries. Only verifier
+timeouts retry, with at most three attempts on the same output and the original
+900-second limit (1,800 for scheduler). All jobs use the standard Harbor runner.
+
+## Official outcomes
+
+Cells list repeat 1 / repeat 2. `E (raw 0)` means verifier dependencies failed
+before tests started; the native zero remains recorded but is not a capability
+failure.
+
+| Task | Cohort | Control | Verification | Operations |
+| --- | --- | --- | --- | --- |
+| `torch-pipeline-parallelism` | Diagnostic | E (raw 0) / E (raw 0) | Fail / Pass | Fail / Fail |
+| `llm-inference-batching-scheduler` | Diagnostic | Fail / Pass | Pass / Pass | Pass / Pass |
+| `build-cython-ext` | Outside prior tuning | Pass / Pass | Pass / Pass | Pass / Pass |
+| `cancel-async-tasks` | Outside prior tuning | Pass / Pass | Pass / Pass | Pass / Pass |
+
+Across all valid scores, control is 5/6 with two verifier errors, verification
+7/8, and operations 6/8. **Those denominators differ.** The common comparison
+excludes both pipeline repeats from every arm: control 5/6 versus 6/6 for each
+candidate, with five shared passes and one candidate-only pass. No fresh paired
+pipeline score is available. Both held-out tasks pass in every arm and repeat.
+Cython verifiers each report 11 passed; async verifiers each report 6 passed.
+
+### Why the valid failures occurred
+
+- **Control scheduler, repeat 1:** the official verifier reports one failure and
+  five passes. Each output has eight shapes, but their combined union has
+  **eleven**, violating the public limit of eight across both files. The agent
+  rechecks final files, metrics and input hashes but omits the global union.
+  Its final completion claim therefore exceeds its evidence. The other five
+  scheduler trials explicitly check the union and pass all six official tests.
+- **Verification pipeline, repeat 1:** Torch never becomes available during
+  agent work. The agent writes a real-model harness but only executes syntax
+  validation, disclosing the gap. Official tests then fail single-rank backward
+  activation comparison at `lm_head` (maximum difference about 0.5714225) and
+  two-rank execution because `local_targets[micro]` is `None`. Its chosen loss
+  scaling and backward order were never numerically validated.
+- **Operations pipeline, both repeats:** official tests execute after one
+  download-timeout retry per trial; both report two failures and two passes.
+  Single-rank `lm_head.bwd` / two-rank `model.layers.1.bwd` differences are about
+  0.1428563 / 0.0421933 in repeat 1 and 0.0416673 / 0.0146767 in repeat 2.
+  Repeat 1 never runs its numerical harness. Repeat 2 has passing single-rank
+  parameter checks, but its microbatch-hook comparison fails and its two-rank
+  parameter selection can omit missing gradients. The final discloses the gaps.
+
+These are observed failure locations and verification omissions, not uniquely
+proven causal decompositions. No counterfactual patch-and-rerun was performed.
+
+### Why the two control pipeline zeros are excluded
+
+Repeat 1 times out twice at 900 seconds. Attempt 3 returns after a network
+failure downloading/extracting `nvidia-cufft-cu12`; the unchanged test script
+writes zero although pytest never starts. Repeat 2 cannot fetch the pytest
+package index and also never starts tests. That failure is not a verifier
+timeout, so it receives no retry. Harbor reports native scored zeros without
+exceptions; the analytical classification is `verifier_error` for both.
+
+[Preflight evidence](../results/verification-quality-preflight-20261009.json)
+records six successful agent installations before model work. All selected
+verifier dependency profiles remain **`not_configured`**. These installation
+checks did not certify verifier dependencies. No extra attempts were added to
+repair these comparison gaps.
+
+## What changed in verification behavior
+
+### Useful evidence from the verification prompt
+
+Five of eight verification trajectories show a deliberate negative control,
+versus none observed in the other arms. Their strength differs:
+
+| Trial | Observed discrimination | Limit of the evidence |
+| --- | --- | --- |
+| Scheduler repeat 1 | Provided baseline fails three metric thresholds in both buckets | Cost/threshold logic only; not the complete structural checker despite the final wording |
+| Scheduler repeat 2 | Same validator rejects four malformed plan copies | Selected coverage/alignment/ID mutations; not every global constraint |
+| Async repeat 1 | Same suites reject naive gather and cancel-without-wait runners | Selected cancellation and SIGINT entry points |
+| Async repeat 2 | Same real-SIGINT check rejects double-cancel cleanup | One deliberately invalid runner |
+| Cython repeat 2 | Numerical comparison rejects a perturbed contribution matrix | Printed boolean from a local numeric check; not a faulty extension build |
+
+The async verification runs first expose actual interrupted cleanup, repair the
+implementation, and rerun functional, cancellation and subprocess signal checks
+on final files. This is concrete evidence of useful verification work, even
+though control and operations also pass the official async tests.
+
+### Remaining gaps, including official passes
+
+**Pipeline verification repeat 2 passes all four official tests but does not
+establish all its own claims.** Real-model checks identify broadcast-result and
+input-argument bugs, which the agent repairs. It then fails tied embedding/head
+parameter checks on two ranks and changes the harness to untied weights without
+resolving the tied case. It skips unobserved layers without an independent
+partition assertion. An intermediate claim of parameter agreement contradicts
+observed output. The final model call exhausts the work budget, leaving no final
+summary or disclosure. A zero count of unsupported *final* claims in this arm
+must not be interpreted as flawless reporting.
+
+**Operations Cython repeat 2 passes the official verifier but overstates its
+self-check.** Its last numerical probe prints `False` for the no-basepoint
+comparison and `True` for basepoint. The reference applies the basepoint formula
+to both; the agent neither fixes and reruns that oracle nor discloses the
+mismatch in its final agreement summary. The error is in the evidence, not a
+proven failure of the installed extension. Operations repeat 1 discloses a
+broader smoke-test failure, but calls the process still running after a recorded
+exit status of 1.
+
+**Operations async repeat 1** verifies direct `asyncio.run` plus real SIGINT and
+awaited cleanup, but an outer-gather case still interrupts cleanup. The final
+discloses the limitation; its assertion that every coroutine runner shares it
+is not established by these traces.
+
+**Control pipeline repeat 2** has strong small real-model checks of activations,
+owned parameters and loss. After a compatibility fix it repeats two Transformers
+versions, but not the third version included in its final summary. That is one
+observed stale verification claim. Cython control repeat 2 removes an invented
+figure-eight constant after a mismatch while retaining the Python comparison;
+verification repeat 2 broadens a similar constant set. Their remaining checks
+are useful, but neither change establishes a case-specific independent constant.
+
+All Cython arms execute the required README example, confirm real compiled
+modules in system site-packages with NumPy 2.3.0, and pass the 18-test public
+suite using only the two authorized file exclusions. No edits to those public
+test assertions were observed. Many probes use `tail`/`grep` without `pipefail`:
+we credit visible test results and assertions, not shell exit status alone.
+
+### Pipeline contract caveat
+
+After all six pipeline outputs froze, the known diagnostic verifier was reviewed
+again. It zips hook records in call order against reference microbatches in input
+order. Public AFAB wording does not specify backward microbatch order. A
+self-check that aligns by microbatch identity can validate numerical values while
+violating that ordering constraint. This caveat does not make partial self-tests
+sufficient, nor establish that order alone explains every failure. No verifier
+or task file was modified.
+
+## Cost and execution time
+
+Agent seconds are Harbor execution intervals including tools and finalization,
+excluding setup, verifier and queue time. Costs are token-based estimates, not
+provider invoices. `>=` marks a known lower bound due to incomplete accounting.
+
+| All eight trials per arm | Control | Verification | Operations |
+| --- | ---: | ---: | ---: |
+| Estimated USD | >= 0.633712 | >= 0.750448 | 0.598339 |
+| Agent seconds | 3,777.7 | 4,124.5 | 3,809.1 |
+| Model attempts | 311 | 334 | 272 |
+| Incomplete model attempts / accounting trials | 1 | 2 | 0 |
+| Bash tool timeouts | 1 | 2 | 3 |
+| Truncated responses | 0 | 0 | 0 |
+
+| Six common eligible cases per arm | Control | Verification | Operations |
+| --- | ---: | ---: | ---: |
+| Passes | 5/6 | 6/6 | 6/6 |
+| Estimated USD | >= 0.478366 | >= 0.585636 | 0.462751 |
+| Agent seconds | 2,519.0 | 2,747.9 | 2,497.8 |
+| Model attempts | 258 | 292 | 229 |
+
+Verification uses **9.1% more agent time** on the common cases. Total-cost
+percentage differences are not identifiable from the incomplete estimates.
+For a descriptive accounting-complete subset of five common cases (exclude
+Cython repeat 2 from all arms), costs are $0.411074 control, $0.500304 verification
+and $0.395594 operations. This post hoc accounting subset is not a replacement
+score cohort.
+
+The three accounting gaps are control Cython repeat 2 and verification Cython
+repeat 2 (recovered incomplete network streams), plus verification pipeline
+repeat 2 (final-call deadline). Completed retries do not recover missing usage
+for the interrupted requests. All other model runs finish with a final response.
+
+The operations arm still has two pipeline background-command timeouts and one
+async shell-signal-driver timeout. The pipeline launches leave `cd ... &&`
+outside the redirected group, retaining captured pipes; the async trial later
+succeeds with a Python subprocess driver. Prompt instructions alone therefore
+do not provide reliable process lifecycle management.
+
+## Integrity, limitations and next experiment
+
+Final checks validate all 24 ATIF trajectories, recorded runtime source hashes,
+installed wheel hashes, frozen task/config/prompt hashes, and the four original
+image IDs. Every verifier obeys the timeout-only retry policy. All 18 non-pipeline
+verifier summaries execute tests; there are no native execution exceptions.
+The complete audit finds no confirmed external benchmark-answer/hidden-test
+exposure. One pipeline trial attempts exact-function searches without obtaining
+an answer. Cython verification repeat 1 consults two upstream files in the
+explicitly task-supplied public project; this is recorded as an allowed code
+reference, not benchmark-answer retrieval.
+
+Pipeline and scheduler are selected diagnostic tasks, and the designer had
+previously seen pipeline evaluation code. Candidate prompts contain no hidden
+expected answer or task-specific implementation rule. The two new tasks were
+selected from catalog names/public requirements; their hidden tests and solutions
+were not inspected before outputs froze. Two repeats, two new tasks, an unblinded
+manual audit, uncontrolled network/cache/provider variation, and missing valid
+pipeline controls prevent a general causal or statistical claim.
+
+The next implementation priorities are:
+
+1. **Map requirements to evidence, including global constraints.** Track each
+   high-risk requirement's scope (item, file, cross-file), observed check result,
+   and final artifact version. The scheduler failure shows why per-file checks
+   cannot establish a cross-file requirement.
+2. **Preserve failed cases and verify the checker.** Prefer a small real entry
+   point plus an independent oracle. Keep failing cases visible when correcting
+   tests; record why an expected value or coverage selection changes. Use a cheap
+   invalid output/implementation for critical custom checks. Tied-to-untied
+   substitution must remain an unresolved gap.
+3. **Tie completion claims to actual results.** Rerun affected checks after final
+   edits; distinguish pass, fail, not run and stale evidence. Preserve the exit
+   status of the tested process through pipes. Reserve time for a final evidence
+   summary rather than expanding unrelated smoke coverage until the deadline.
+4. **Provide managed long-command execution.** Return a process handle, pollable
+   status, bounded logs and the actual exit code. Evaluate this runtime change
+   separately from verification wording; the operations prompt did not solve it.
+5. **Make verifier setup reliable before another comparison.** Add reviewed
+   task-revision dependency profiles and tests through the standard adapter.
+   Keep task images, package sources and official scripts intact, and report
+   infrastructure failure separately from task correctness.
+
+These are follow-up directions, not additional changes tested in this run. A new
+preregistered comparison should use fresh task/repeat samples and valid shared
+controls; retain this run unchanged and report unsuccessful trials alongside
+wins.
+
+## Repository validation
+
+The result collector fails closed on incomplete audits, changed evidence hashes,
+changed frozen sources, wheel mismatches or invalid verifier retry policies. It
+preserves native zeros and partial accounting, and computes common eligible
+cases separately from all-trial totals. Harbor regression tests: **105 passed**,
+including six new collection tests. The previously tested frozen prompt/runtime
+change passed **351 core tests, 4 skipped**. Research notes in both languages are
+unchanged. Draft PR #48 retains the experimental status; this report does not
+recommend merging the candidate into main.
diff --git a/benchmarks/harbor/results/verification-quality-20261009.json b/benchmarks/harbor/results/verification-quality-20261009.json
index b3f4a16..3b0339a 100644
--- a/benchmarks/harbor/results/verification-quality-20261009.json
+++ b/benchmarks/harbor/results/verification-quality-20261009.json
@@ -1,56 +1,187 @@
 {
-  "schema_version": 1,
-  "generated_at": "2026-10-09T16:42:31.828366+00:00",
+  "schema_version": 2,
+  "status": "completed",
+  "generated_at": "2026-10-11T14:04:26.098862+00:00",
   "experiment_plan": "benchmarks/harbor/configs/verification-quality-20261009/experiment.json",
   "experiment_plan_sha256": "b3968a10a29507b72c62f6f886a6af4f982373e15ca6103da21dd288362396c1",
-  "status": "running",
+  "audit_path": "benchmarks/harbor/results/verification-quality-audit-20261009.json",
+  "audit_sha256": "a481802413ad300df76d170275dfad2b3e412f8358d46e62ab00f0fa79d3aadf",
+  "cost_note": "Token-based estimates, not invoices. Incomplete totals are known lower bounds, never zero-imputed.",
   "summary": {
     "control": {
-      "planned": 8,
+      "trials": 8,
       "statuses": {
         "verifier_error": 2,
-        "not_started": 6
+        "scored": 6
       },
-      "passes": 0,
-      "scored_failures": 0,
-      "known_cost_usd": 0.155345088,
+      "passes": 5,
+      "scored_failures": 1,
+      "known_cost_usd": 0.633711552,
       "cost_complete": false,
       "usage_complete": false,
-      "known_agent_seconds": 1258.6505069999998,
-      "model_attempts": 53,
+      "incomplete_accounting_trials": 1,
+      "agent_execution_seconds": 3777.668552,
+      "model_attempts": 311,
+      "incomplete_model_attempts": 1,
       "tool_timeouts": 1
     },
     "verification": {
-      "planned": 8,
+      "trials": 8,
       "statuses": {
-        "scored": 2,
-        "started": 1,
-        "not_started": 5
+        "scored": 8
       },
-      "passes": 1,
+      "passes": 7,
       "scored_failures": 1,
-      "known_cost_usd": 0.164811612,
+      "known_cost_usd": 0.750447996,
       "cost_complete": false,
       "usage_complete": false,
-      "known_agent_seconds": 1376.596241,
-      "model_attempts": 42,
+      "incomplete_accounting_trials": 2,
+      "agent_execution_seconds": 4124.486748,
+      "model_attempts": 334,
+      "incomplete_model_attempts": 2,
       "tool_timeouts": 2
     },
     "operations": {
-      "planned": 8,
+      "trials": 8,
       "statuses": {
-        "scored": 2,
-        "started": 1,
-        "not_started": 5
+        "scored": 8
       },
-      "passes": 0,
+      "passes": 6,
       "scored_failures": 2,
-      "known_cost_usd": 0.135588492,
+      "known_cost_usd": 0.598339164,
+      "cost_complete": true,
+      "usage_complete": true,
+      "incomplete_accounting_trials": 0,
+      "agent_execution_seconds": 3809.063739,
+      "model_attempts": 272,
+      "incomplete_model_attempts": 0,
+      "tool_timeouts": 3
+    }
+  },
+  "common_eligible_cases": [
+    {
+      "task": "terminal-bench/build-cython-ext",
+      "repeat": 1
+    },
+    {
+      "task": "terminal-bench/build-cython-ext",
+      "repeat": 2
+    },
+    {
+      "task": "terminal-bench/cancel-async-tasks",
+      "repeat": 1
+    },
+    {
+      "task": "terminal-bench/cancel-async-tasks",
+      "repeat": 2
+    },
+    {
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "repeat": 1
+    },
+    {
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "repeat": 2
+    }
+  ],
+  "common_eligible_summary": {
+    "control": {
+      "trials": 6,
+      "statuses": {
+        "scored": 6
+      },
+      "passes": 5,
+      "scored_failures": 1,
+      "known_cost_usd": 0.478366464,
       "cost_complete": false,
       "usage_complete": false,
-      "known_agent_seconds": 1311.230174,
-      "model_attempts": 43,
-      "tool_timeouts": 2
+      "incomplete_accounting_trials": 1,
+      "agent_execution_seconds": 2519.018045,
+      "model_attempts": 258,
+      "incomplete_model_attempts": 1,
+      "tool_timeouts": 0
+    },
+    "verification": {
+      "trials": 6,
+      "statuses": {
+        "scored": 6
+      },
+      "passes": 6,
+      "scored_failures": 0,
+      "known_cost_usd": 0.585636384,
+      "cost_complete": false,
+      "usage_complete": false,
+      "incomplete_accounting_trials": 1,
+      "agent_execution_seconds": 2747.890507,
+      "model_attempts": 292,
+      "incomplete_model_attempts": 1,
+      "tool_timeouts": 0
+    },
+    "operations": {
+      "trials": 6,
+      "statuses": {
+        "scored": 6
+      },
+      "passes": 6,
+      "scored_failures": 0,
+      "known_cost_usd": 0.462750672,
+      "cost_complete": true,
+      "usage_complete": true,
+      "incomplete_accounting_trials": 0,
+      "agent_execution_seconds": 2497.833565,
+      "model_attempts": 229,
+      "incomplete_model_attempts": 0,
+      "tool_timeouts": 1
+    }
+  },
+  "held_out_common_summary": {
+    "control": {
+      "trials": 4,
+      "statuses": {
+        "scored": 4
+      },
+      "passes": 4,
+      "scored_failures": 0,
+      "known_cost_usd": 0.27104898,
+      "cost_complete": false,
+      "usage_complete": false,
+      "incomplete_accounting_trials": 1,
+      "agent_execution_seconds": 1337.221293,
+      "model_attempts": 190,
+      "incomplete_model_attempts": 1,
+      "tool_timeouts": 0
+    },
+    "verification": {
+      "trials": 4,
+      "statuses": {
+        "scored": 4
+      },
+      "passes": 4,
+      "scored_failures": 0,
+      "known_cost_usd": 0.338877636,
+      "cost_complete": false,
+      "usage_complete": false,
+      "incomplete_accounting_trials": 1,
+      "agent_execution_seconds": 1533.798879,
+      "model_attempts": 229,
+      "incomplete_model_attempts": 1,
+      "tool_timeouts": 0
+    },
+    "operations": {
+      "trials": 4,
+      "statuses": {
+        "scored": 4
+      },
+      "passes": 4,
+      "scored_failures": 0,
+      "known_cost_usd": 0.230231148,
+      "cost_complete": true,
+      "usage_complete": true,
+      "incomplete_accounting_trials": 0,
+      "agent_execution_seconds": 1589.210119,
+      "model_attempts": 173,
+      "incomplete_model_attempts": 0,
+      "tool_timeouts": 1
     }
   },
   "trials": [
@@ -61,27 +192,51 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
       "job_name": "vq-con-torch-pipeline-parallelism-r1-20261009",
       "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
+      "cohort": "diagnostic",
+      "native_status": "scored",
+      "native_reward": 0.0,
+      "raw_verifier_rewards": {
+        "reward": 0.0
+      },
+      "exception_type": null,
       "status": "verifier_error",
       "reward": null,
-      "launcher_exit_code": 0,
+      "eligible_as_independent_capability_evidence": false,
       "known_cost_usd": 0.085583436,
       "cost_complete": true,
       "usage_complete": true,
-      "raw_verifier_rewards": {
-        "reward": 0.0
+      "metrics": {
+        "total_steps": 55,
+        "total_prompt_tokens": 1230161,
+        "total_completion_tokens": 56274,
+        "total_cached_tokens": 1193856,
+        "total_cost_usd": 0.085583436,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.085583436
+        }
       },
-      "exception_type": null,
+      "launcher_exit_code": 0,
       "result_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__YcctKv5/result.json",
+      "result_sha256": "c5d0202f86b7ecdff97de8343b13d69e46484238a28cf569d5700dbc8de65b66",
       "task_checksum": "31e565558cd22598a3c2069f771d9d0eee26fb843b77f7b03e00ad4c336fbce9",
-      "agent_setup_seconds": 137.350091,
-      "agent_execution_seconds": 671.625725,
-      "verifier_seconds": 2345.913464,
-      "verifier_preflight": {
-        "profile": null,
-        "status": "not_configured"
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__YcctKv5/agent/trajectory.json",
+      "trajectory_sha256": "698e12648ccb5f0f8312d6ee8565d7cec6185f5aa535ab21db847bcb03b08cc2",
+      "verifier_stdout_sha256": "652980f8a03f05e73bd524cd4fe42bc27c2f957c4ddb2c7ebec3ffed4d44b6e7",
+      "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
+      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 671658.796661,
+        "timestamp": "2026-10-09T15:05:36.416Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
       },
       "started_at": "2026-10-09T14:52:06.449794Z",
       "finished_at": "2026-10-09T15:44:55.652186Z",
+      "agent_setup_seconds": 137.350091,
+      "agent_execution_seconds": 671.625725,
+      "verifier_seconds": 2345.913464,
       "verifier_attempts": {
         "timeout_sec": 900.0,
         "max_attempts": 3,
@@ -104,36 +259,14 @@
           }
         ]
       },
-      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__YcctKv5/agent/trajectory.json",
-      "trajectory_sha256": "698e12648ccb5f0f8312d6ee8565d7cec6185f5aa535ab21db847bcb03b08cc2",
-      "terminal": {
-        "status": "completed",
-        "duration_ms": 671658.796661,
-        "timestamp": "2026-10-09T15:05:36.416Z",
-        "timestamp_source": "source_timestamp",
-        "outcome": "completed"
-      },
-      "metrics": {
-        "total_steps": 55,
-        "total_prompt_tokens": 1230161,
-        "total_completion_tokens": 56274,
-        "total_cached_tokens": 1193856,
-        "total_cost_usd": 0.085583436,
-        "extra": {
-          "cost_is_estimated": true,
-          "estimated_cost_usd": 0.085583436
-        }
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
       },
       "model_attempts": 27,
-      "summed_model_call_seconds": 316.959456431,
-      "maximum_model_call_seconds": 73.936952907,
-      "summed_tool_call_seconds": 354.618524057,
       "incomplete_model_attempts": 0,
       "truncated_responses": 0,
-      "recovery_events": 0,
       "tool_timeouts": 1,
-      "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
-      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350",
       "audit": {
         "job": "vq-con-torch-pipeline-parallelism-r1-20261009",
         "arm": "control",
@@ -163,11 +296,10 @@
         "score_valid_for_capability_comparison": false,
         "score_classification": "verifier_dependency_error",
         "official_failure_notes": "The first two verifier attempts timed out at 900 seconds each. Attempt 3 returned normally after uv failed to download/extract nvidia-cufft-cu12 because of a network timeout. The unchanged task script wrote reward 0 despite never starting pytest. Preserve raw reward 0; exclude this trial from capability-score comparisons and count it as a verifier dependency error, not an agent-scored failure.",
-        "exclusion_reason": "verifier_dependency_error"
-      },
-      "native_status": "scored",
-      "native_reward": 0.0,
-      "failure_stage": "verifier_dependencies"
+        "exclusion_reason": "verifier_dependency_error",
+        "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__YcctKv5/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "652980f8a03f05e73bd524cd4fe42bc27c2f957c4ddb2c7ebec3ffed4d44b6e7"
+      }
     },
     {
       "arm": "verification",
@@ -176,27 +308,51 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
       "job_name": "vq-ver-torch-pipeline-parallelism-r1-20261009",
       "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
+      "cohort": "diagnostic",
+      "native_status": "scored",
+      "native_reward": 0.0,
+      "raw_verifier_rewards": {
+        "reward": 0.0
+      },
+      "exception_type": null,
       "status": "scored",
       "reward": 0.0,
-      "launcher_exit_code": 0,
+      "eligible_as_independent_capability_evidence": true,
       "known_cost_usd": 0.0612708,
       "cost_complete": true,
       "usage_complete": true,
-      "raw_verifier_rewards": {
-        "reward": 0.0
+      "metrics": {
+        "total_steps": 41,
+        "total_prompt_tokens": 600080,
+        "total_completion_tokens": 46079,
+        "total_cached_tokens": 592000,
+        "total_cost_usd": 0.0612708,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.0612708
+        }
       },
-      "exception_type": null,
+      "launcher_exit_code": 0,
       "result_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__TxDd3vH/result.json",
+      "result_sha256": "6346e793b52c5ab904101d340d7efe443d8ed081d5fe14150783b4be89b6dc78",
       "task_checksum": "31e565558cd22598a3c2069f771d9d0eee26fb843b77f7b03e00ad4c336fbce9",
-      "agent_setup_seconds": 77.230504,
-      "agent_execution_seconds": 655.750458,
-      "verifier_seconds": 703.878173,
-      "verifier_preflight": {
-        "profile": null,
-        "status": "not_configured"
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__TxDd3vH/agent/trajectory.json",
+      "trajectory_sha256": "c47b1502cc633647967ba9b7fbd7a99c9528a35180e7e826a6d6bb76b5f938d8",
+      "verifier_stdout_sha256": "2e5ed3211a3771b61deb6d8510b63d3ce2ddc03157d0ff1888ca31aa4d652d21",
+      "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
+      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 655947.282239,
+        "timestamp": "2026-10-09T15:04:20.413Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
       },
       "started_at": "2026-10-09T14:52:06.479199Z",
       "finished_at": "2026-10-09T15:16:18.452768Z",
+      "agent_setup_seconds": 77.230504,
+      "agent_execution_seconds": 655.750458,
+      "verifier_seconds": 703.878173,
       "verifier_attempts": {
         "timeout_sec": 900.0,
         "max_attempts": 3,
@@ -209,36 +365,14 @@
           }
         ]
       },
-      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__TxDd3vH/agent/trajectory.json",
-      "trajectory_sha256": "c47b1502cc633647967ba9b7fbd7a99c9528a35180e7e826a6d6bb76b5f938d8",
-      "terminal": {
-        "status": "completed",
-        "duration_ms": 655947.282239,
-        "timestamp": "2026-10-09T15:04:20.413Z",
-        "timestamp_source": "source_timestamp",
-        "outcome": "completed"
-      },
-      "metrics": {
-        "total_steps": 41,
-        "total_prompt_tokens": 600080,
-        "total_completion_tokens": 46079,
-        "total_cached_tokens": 592000,
-        "total_cost_usd": 0.0612708,
-        "extra": {
-          "cost_is_estimated": true,
-          "estimated_cost_usd": 0.0612708
-        }
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
       },
       "model_attempts": 20,
-      "summed_model_call_seconds": 248.364908082,
-      "maximum_model_call_seconds": 61.362616916,
-      "summed_tool_call_seconds": 407.52567982,
       "incomplete_model_attempts": 0,
       "truncated_responses": 0,
-      "recovery_events": 0,
       "tool_timeouts": 2,
-      "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
-      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c",
       "audit": {
         "job": "vq-ver-torch-pipeline-parallelism-r1-20261009",
         "arm": "verification",
@@ -270,7 +404,11 @@
             "note": "Exact-function searches return security challenges, missing endpoints, authentication errors, or no answer links. Two Bash calls time out at 120 seconds. Torch remains unavailable; a real-model harness is written but not executed. Only py_compile passes, and the final explicitly discloses no numerical verification and the chosen loss-scale interpretation."
           }
         ],
-        "official_failure_notes": "Completed verifier reward 0: world_size=1 backward activation mismatch at lm_head (max diff 0.5714225); world_size=2 raises AttributeError because local_targets[micro] is None. The candidate chose multiplication by the microbatch count and did not run its numerical harness."
+        "official_failure_notes": "Completed verifier reward 0: world_size=1 backward activation mismatch at lm_head (max diff 0.5714225); world_size=2 raises AttributeError because local_targets[micro] is None. The candidate chose multiplication by the microbatch count and did not run its numerical harness.",
+        "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__TxDd3vH/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "2e5ed3211a3771b61deb6d8510b63d3ce2ddc03157d0ff1888ca31aa4d652d21",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_failure"
       }
     },
     {
@@ -280,27 +418,51 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
       "job_name": "vq-ope-torch-pipeline-parallelism-r1-20261009",
       "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
+      "cohort": "diagnostic",
+      "native_status": "scored",
+      "native_reward": 0.0,
+      "raw_verifier_rewards": {
+        "reward": 0.0
+      },
+      "exception_type": null,
       "status": "scored",
       "reward": 0.0,
-      "launcher_exit_code": 0,
+      "eligible_as_independent_capability_evidence": true,
       "known_cost_usd": 0.076849608,
       "cost_complete": true,
       "usage_complete": true,
-      "raw_verifier_rewards": {
-        "reward": 0.0
+      "metrics": {
+        "total_steps": 41,
+        "total_prompt_tokens": 711314,
+        "total_completion_tokens": 49473,
+        "total_cached_tokens": 666368,
+        "total_cost_usd": 0.076849608,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.076849608
+        }
       },
-      "exception_type": null,
+      "launcher_exit_code": 0,
       "result_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__bEqs3hp/result.json",
+      "result_sha256": "fe63e0c2cd5f3b89804c09423762b245a6bb9c821c517f21a66da492d40d2e74",
       "task_checksum": "31e565558cd22598a3c2069f771d9d0eee26fb843b77f7b03e00ad4c336fbce9",
-      "agent_setup_seconds": 134.611081,
-      "agent_execution_seconds": 655.875226,
-      "verifier_seconds": 1252.528498,
-      "verifier_preflight": {
-        "profile": null,
-        "status": "not_configured"
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__bEqs3hp/agent/trajectory.json",
+      "trajectory_sha256": "96d56bbfa98b0e779517f6a4a5c85cfd70d417e2dbe9c318aa343bf814be3807",
+      "verifier_stdout_sha256": "b5b8a9d7d518a129335b76707c8d9fb1dbea7172228ac9bd9bc4251eeb7e28c5",
+      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
+      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 653746.844654,
+        "timestamp": "2026-10-09T15:58:39.067Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
       },
       "started_at": "2026-10-09T15:45:27.637317Z",
       "finished_at": "2026-10-09T16:19:45.390521Z",
+      "agent_setup_seconds": 134.611081,
+      "agent_execution_seconds": 655.875226,
+      "verifier_seconds": 1252.528498,
       "verifier_attempts": {
         "timeout_sec": 900.0,
         "max_attempts": 3,
@@ -318,36 +480,14 @@
           }
         ]
       },
-      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__bEqs3hp/agent/trajectory.json",
-      "trajectory_sha256": "96d56bbfa98b0e779517f6a4a5c85cfd70d417e2dbe9c318aa343bf814be3807",
-      "terminal": {
-        "status": "completed",
-        "duration_ms": 653746.844654,
-        "timestamp": "2026-10-09T15:58:39.067Z",
-        "timestamp_source": "source_timestamp",
-        "outcome": "completed"
-      },
-      "metrics": {
-        "total_steps": 41,
-        "total_prompt_tokens": 711314,
-        "total_completion_tokens": 49473,
-        "total_cached_tokens": 666368,
-        "total_cost_usd": 0.076849608,
-        "extra": {
-          "cost_is_estimated": true,
-          "estimated_cost_usd": 0.076849608
-        }
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
       },
       "model_attempts": 20,
-      "summed_model_call_seconds": 243.921774978,
-      "maximum_model_call_seconds": 167.232692041,
-      "summed_tool_call_seconds": 409.771602606,
       "incomplete_model_attempts": 0,
       "truncated_responses": 0,
-      "recovery_events": 0,
       "tool_timeouts": 1,
-      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
-      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc",
       "audit": {
         "job": "vq-ope-torch-pipeline-parallelism-r1-20261009",
         "arm": "operations",
@@ -392,27 +532,51 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
       "job_name": "vq-ope-torch-pipeline-parallelism-r2-20261009",
       "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
+      "cohort": "diagnostic",
+      "native_status": "scored",
+      "native_reward": 0.0,
+      "raw_verifier_rewards": {
+        "reward": 0.0
+      },
+      "exception_type": null,
       "status": "scored",
       "reward": 0.0,
-      "launcher_exit_code": 0,
+      "eligible_as_independent_capability_evidence": true,
       "known_cost_usd": 0.058738884,
       "cost_complete": true,
       "usage_complete": true,
-      "raw_verifier_rewards": {
-        "reward": 0.0
+      "metrics": {
+        "total_steps": 47,
+        "total_prompt_tokens": 724439,
+        "total_completion_tokens": 43079,
+        "total_cached_tokens": 715264,
+        "total_cost_usd": 0.058738884,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.058738884
+        }
       },
-      "exception_type": null,
+      "launcher_exit_code": 0,
       "result_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__a3m3xr8/result.json",
+      "result_sha256": "b0b3fe5a21a4cdf69ea39887370b48c99b11ff684b2186bd3f4c3a411cfe888e",
       "task_checksum": "31e565558cd22598a3c2069f771d9d0eee26fb843b77f7b03e00ad4c336fbce9",
-      "agent_setup_seconds": 138.896958,
-      "agent_execution_seconds": 655.354948,
-      "verifier_seconds": 1244.67741,
-      "verifier_preflight": {
-        "profile": null,
-        "status": "not_configured"
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__a3m3xr8/agent/trajectory.json",
+      "trajectory_sha256": "6e4fc530b2124cd82e7f9b41c9b12281ed77eb0a3c49d898494a68f27437fbdd",
+      "verifier_stdout_sha256": "54ed7253a7377c20b895455196ab02ed8077fcd34a7465d6f9e99485b6dc5396",
+      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
+      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 653173.203504,
+        "timestamp": "2026-10-09T15:58:34.005Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
       },
       "started_at": "2026-10-09T15:45:18.428047Z",
       "finished_at": "2026-10-09T16:19:32.758980Z",
+      "agent_setup_seconds": 138.896958,
+      "agent_execution_seconds": 655.354948,
+      "verifier_seconds": 1244.67741,
       "verifier_attempts": {
         "timeout_sec": 900.0,
         "max_attempts": 3,
@@ -430,36 +594,14 @@
           }
         ]
       },
-      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__a3m3xr8/agent/trajectory.json",
-      "trajectory_sha256": "6e4fc530b2124cd82e7f9b41c9b12281ed77eb0a3c49d898494a68f27437fbdd",
-      "terminal": {
-        "status": "completed",
-        "duration_ms": 653173.203504,
-        "timestamp": "2026-10-09T15:58:34.005Z",
-        "timestamp_source": "source_timestamp",
-        "outcome": "completed"
-      },
-      "metrics": {
-        "total_steps": 47,
-        "total_prompt_tokens": 724439,
-        "total_completion_tokens": 43079,
-        "total_cached_tokens": 715264,
-        "total_cost_usd": 0.058738884,
-        "extra": {
-          "cost_is_estimated": true,
-          "estimated_cost_usd": 0.058738884
-        }
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
       },
       "model_attempts": 23,
-      "summed_model_call_seconds": 235.563146319,
-      "maximum_model_call_seconds": 106.58585946199999,
-      "summed_tool_call_seconds": 417.548233131,
       "incomplete_model_attempts": 0,
       "truncated_responses": 0,
-      "recovery_events": 0,
       "tool_timeouts": 1,
-      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
-      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc",
       "audit": {
         "job": "vq-ope-torch-pipeline-parallelism-r2-20261009",
         "arm": "operations",
@@ -507,27 +649,50 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
       "job_name": "vq-ver-torch-pipeline-parallelism-r2-20261009",
       "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
+      "cohort": "diagnostic",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
       "status": "scored",
       "reward": 1.0,
-      "launcher_exit_code": 0,
+      "eligible_as_independent_capability_evidence": true,
       "known_cost_usd": 0.103540812,
       "cost_complete": false,
       "usage_complete": false,
-      "raw_verifier_rewards": {
-        "reward": 1.0
+      "metrics": {
+        "total_steps": 45,
+        "extra": {
+          "usage_complete": false,
+          "known_cost_usd": 0.103540812,
+          "cost_is_partial": true,
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.103540812
+        }
       },
-      "exception_type": null,
+      "launcher_exit_code": 0,
       "result_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__TUMs5AX/result.json",
+      "result_sha256": "9290474e8d65e7a663efb05c705d13e3a2ad632b16795cd4a2a2021f880ab867",
       "task_checksum": "31e565558cd22598a3c2069f771d9d0eee26fb843b77f7b03e00ad4c336fbce9",
-      "agent_setup_seconds": 104.088802,
-      "agent_execution_seconds": 720.845783,
-      "verifier_seconds": 463.852849,
-      "verifier_preflight": {
-        "profile": null,
-        "status": "not_configured"
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__TUMs5AX/agent/trajectory.json",
+      "trajectory_sha256": "db9d1cb521b78bcdd040b07393c605f8bbbcdd0bf3f299d7c52eaf34a2563a2b",
+      "verifier_stdout_sha256": "931c5d89033bae35cf65965dd76519f77eb2b899ac50ebbfc96564fd755e4e73",
+      "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
+      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 720014.652115,
+        "timestamp": "2026-10-09T16:33:43.843Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "time_budget_exhausted"
       },
       "started_at": "2026-10-09T16:19:57.988835Z",
       "finished_at": "2026-10-09T16:41:41.882654Z",
+      "agent_setup_seconds": 104.088802,
+      "agent_execution_seconds": 720.845783,
+      "verifier_seconds": 463.852849,
       "verifier_attempts": {
         "timeout_sec": 900.0,
         "max_attempts": 3,
@@ -540,35 +705,14 @@
           }
         ]
       },
-      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__TUMs5AX/agent/trajectory.json",
-      "trajectory_sha256": "db9d1cb521b78bcdd040b07393c605f8bbbcdd0bf3f299d7c52eaf34a2563a2b",
-      "terminal": {
-        "status": "completed",
-        "duration_ms": 720014.652115,
-        "timestamp": "2026-10-09T16:33:43.843Z",
-        "timestamp_source": "source_timestamp",
-        "outcome": "time_budget_exhausted"
-      },
-      "metrics": {
-        "total_steps": 45,
-        "extra": {
-          "usage_complete": false,
-          "known_cost_usd": 0.103540812,
-          "cost_is_partial": true,
-          "cost_is_estimated": true,
-          "estimated_cost_usd": 0.103540812
-        }
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
       },
       "model_attempts": 22,
-      "summed_model_call_seconds": 493.542580079,
-      "maximum_model_call_seconds": 237.944808967,
-      "summed_tool_call_seconds": 226.41692674,
       "incomplete_model_attempts": 1,
       "truncated_responses": 0,
-      "recovery_events": 0,
       "tool_timeouts": 0,
-      "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
-      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c",
       "audit": {
         "job": "vq-ver-torch-pipeline-parallelism-r2-20261009",
         "arm": "verification",
@@ -624,27 +768,51 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/torch-pipeline-parallelism.json",
       "job_name": "vq-con-torch-pipeline-parallelism-r2-20261009",
       "config_sha256": "a00e2f0c1d42b55c967eb96e4607f733d0290e5461a4bed5e49eb214462751c4",
+      "cohort": "diagnostic",
+      "native_status": "scored",
+      "native_reward": 0.0,
+      "raw_verifier_rewards": {
+        "reward": 0.0
+      },
+      "exception_type": null,
       "status": "verifier_error",
       "reward": null,
-      "launcher_exit_code": 0,
+      "eligible_as_independent_capability_evidence": false,
       "known_cost_usd": 0.069761652,
       "cost_complete": true,
       "usage_complete": true,
-      "raw_verifier_rewards": {
-        "reward": 0.0
+      "metrics": {
+        "total_steps": 53,
+        "total_prompt_tokens": 978643,
+        "total_completion_tokens": 47545,
+        "total_cached_tokens": 955392,
+        "total_cost_usd": 0.069761652,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.069761652
+        }
       },
-      "exception_type": null,
+      "launcher_exit_code": 0,
       "result_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__nuNNDUP/result.json",
+      "result_sha256": "b9319cb895eed8813e60663fcf6b7a49a1d4f45144a5e13fa8082b94e38ca770",
       "task_checksum": "31e565558cd22598a3c2069f771d9d0eee26fb843b77f7b03e00ad4c336fbce9",
-      "agent_setup_seconds": 100.827538,
-      "agent_execution_seconds": 587.024782,
-      "verifier_seconds": 136.906223,
-      "verifier_preflight": {
-        "profile": null,
-        "status": "not_configured"
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__nuNNDUP/agent/trajectory.json",
+      "trajectory_sha256": "c523402ba377e835015d602745083c12d3f8de5a6238ad760072257bad973634",
+      "verifier_stdout_sha256": "6b1a54415cdc40d7b5a80e6a81e00bc6a469d1de80e90bf2e229b1e3528f7c87",
+      "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
+      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 586163.607139,
+        "timestamp": "2026-10-09T16:31:26.693Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
       },
       "started_at": "2026-10-09T16:19:57.731639Z",
       "finished_at": "2026-10-09T16:33:56.359576Z",
+      "agent_setup_seconds": 100.827538,
+      "agent_execution_seconds": 587.024782,
+      "verifier_seconds": 136.906223,
       "verifier_attempts": {
         "timeout_sec": 900.0,
         "max_attempts": 3,
@@ -657,36 +825,14 @@
           }
         ]
       },
-      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__nuNNDUP/agent/trajectory.json",
-      "trajectory_sha256": "c523402ba377e835015d602745083c12d3f8de5a6238ad760072257bad973634",
-      "terminal": {
-        "status": "completed",
-        "duration_ms": 586163.607139,
-        "timestamp": "2026-10-09T16:31:26.693Z",
-        "timestamp_source": "source_timestamp",
-        "outcome": "completed"
-      },
-      "metrics": {
-        "total_steps": 53,
-        "total_prompt_tokens": 978643,
-        "total_completion_tokens": 47545,
-        "total_cached_tokens": 955392,
-        "total_cost_usd": 0.069761652,
-        "extra": {
-          "cost_is_estimated": true,
-          "estimated_cost_usd": 0.069761652
-        }
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
       },
       "model_attempts": 26,
-      "summed_model_call_seconds": 242.30997215399998,
-      "maximum_model_call_seconds": 129.647188506,
-      "summed_tool_call_seconds": 343.783995908,
       "incomplete_model_attempts": 0,
       "truncated_responses": 0,
-      "recovery_events": 0,
       "tool_timeouts": 0,
-      "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
-      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350",
       "audit": {
         "job": "vq-con-torch-pipeline-parallelism-r2-20261009",
         "arm": "control",
@@ -738,10 +884,7 @@
         "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__nuNNDUP/verifier/test-stdout.txt",
         "verifier_stdout_sha256": "6b1a54415cdc40d7b5a80e6a81e00bc6a469d1de80e90bf2e229b1e3528f7c87",
         "final_claim_caveat": "The three-version summary includes a 4.53.3 check performed before the final implementation edit; mark that evidence stale rather than treating the entire completion statement as unsupported."
-      },
-      "native_status": "scored",
-      "native_reward": 0.0,
-      "failure_stage": "verifier_dependencies"
+      }
     },
     {
       "arm": "verification",
@@ -750,58 +893,436 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
       "job_name": "vq-ver-llm-inference-batching-scheduler-r1-20261009",
       "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
-      "status": "started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false,
+      "cohort": "diagnostic",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.09810504,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 63,
+        "total_prompt_tokens": 1679740,
+        "total_completion_tokens": 67304,
+        "total_cached_tokens": 1655040,
+        "total_cost_usd": 0.09810504,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.09810504
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-verification-source/jobs/vq-ver-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__35poZ69/result.json",
+      "result_sha256": "92951249b63b547c8c103286ab6c8ff64db1c5e839f5cb7a12196354e64085cd",
+      "task_checksum": "c19d5af0dc6e54b45f66e3d8f25bfa783c51df3e1844253aa60c20ca0a827046",
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__35poZ69/agent/trajectory.json",
+      "trajectory_sha256": "19a156357a49aa6a68a2073e0065cfe6868e02ee81b2de413291e72ba70f7928",
+      "verifier_stdout_sha256": "17f33b888656af7e23c8f9cb20940fca84289918bd9b24f566b81b3be5828ebb",
       "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
-      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c"
-    },
-    {
-      "arm": "operations",
-      "repeat": 1,
-      "task": "terminal-bench/llm-inference-batching-scheduler",
-      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
-      "job_name": "vq-ope-llm-inference-batching-scheduler-r1-20261009",
-      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
-      "status": "started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false,
-      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
-      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc"
-    },
-    {
-      "arm": "control",
-      "repeat": 1,
-      "task": "terminal-bench/llm-inference-batching-scheduler",
-      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
-      "job_name": "vq-con-llm-inference-batching-scheduler-r1-20261009",
-      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
-    },
-    {
-      "arm": "control",
-      "repeat": 2,
-      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 493519.509075,
+        "timestamp": "2026-10-09T16:51:47.012Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T16:41:54.652748Z",
+      "finished_at": "2026-10-09T16:52:15.225835Z",
+      "agent_setup_seconds": 96.228418,
+      "agent_execution_seconds": 495.142414,
+      "verifier_seconds": 16.302919,
+      "verifier_attempts": {
+        "timeout_sec": 1800.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 15.92654914400191
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 31,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-ver-llm-inference-batching-scheduler-r1-20261009",
+        "arm": "verification",
+        "task": "terminal-bench/llm-inference-batching-scheduler",
+        "repeat": 1,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": null,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": true,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              49,
+              51,
+              53,
+              55,
+              57,
+              59,
+              61,
+              63
+            ],
+            "note": "The final saved plans pass coverage, thresholds and the global union constraint (two shapes). A separate formula implementation agrees with all four cost metrics. The provided baseline is run in a temporary copy and fails three metric thresholds for both buckets. This is a metric negative control, not a run through the complete validation script as the final summary suggests. The final plans and unchanged input hashes are checked again."
+          }
+        ],
+        "sensitivity_scope": "baseline_metric_thresholds",
+        "claim_caveat": "The phrase same checker overstates the negative control: only the cost/threshold logic, not the full structural validator, was exercised.",
+        "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__35poZ69/agent/trajectory.json",
+        "trajectory_sha256": "19a156357a49aa6a68a2073e0065cfe6868e02ee81b2de413291e72ba70f7928",
+        "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__35poZ69/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "17f33b888656af7e23c8f9cb20940fca84289918bd9b24f566b81b3be5828ebb",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "6 passed"
+      }
+    },
+    {
+      "arm": "operations",
+      "repeat": 1,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-ope-llm-inference-batching-scheduler-r1-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
+      "cohort": "diagnostic",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.086658828,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 49,
+        "total_prompt_tokens": 1244237,
+        "total_completion_tokens": 61303,
+        "total_cached_tokens": 1225088,
+        "total_cost_usd": 0.086658828,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.086658828
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-operations-source/jobs/vq-ope-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__Rs4CeJU/result.json",
+      "result_sha256": "151c200d7118f92256de9cdd59613c5d458be8d99d0609b1a1b29806fe9fd451",
+      "task_checksum": "c19d5af0dc6e54b45f66e3d8f25bfa783c51df3e1844253aa60c20ca0a827046",
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__Rs4CeJU/agent/trajectory.json",
+      "trajectory_sha256": "74f2078ba26be1dfb8815d59811dba61af9e2b30afe7f9dbd6422e491f8aadfe",
+      "verifier_stdout_sha256": "75c814a3304cee39967e6c9fc9dfe63b984ffece6ef2860e2e5e8698e3ffa767",
+      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
+      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 373473.09794,
+        "timestamp": "2026-10-09T16:50:14.812Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T16:41:53.038601Z",
+      "finished_at": "2026-10-09T16:50:43.300614Z",
+      "agent_setup_seconds": 125.586824,
+      "agent_execution_seconds": 374.997129,
+      "verifier_seconds": 16.505946,
+      "verifier_attempts": {
+        "timeout_sec": 1800.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 16.149723321002966
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 24,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-ope-llm-inference-batching-scheduler-r1-20261009",
+        "arm": "operations",
+        "task": "terminal-bench/llm-inference-batching-scheduler",
+        "repeat": 1,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": null,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              21,
+              37,
+              41,
+              43,
+              47,
+              49
+            ],
+            "note": "A vectorized metric replica is compared with the public cost model. The actual final files pass metrics, coverage, the global union of seven shapes, and input hashes. A baseline calibration reproduces the poor published metrics, but does not assert that the complete validator rejects a bad plan."
+          }
+        ],
+        "baseline_calibration_observed": true,
+        "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__Rs4CeJU/agent/trajectory.json",
+        "trajectory_sha256": "74f2078ba26be1dfb8815d59811dba61af9e2b30afe7f9dbd6422e491f8aadfe",
+        "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__Rs4CeJU/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "75c814a3304cee39967e6c9fc9dfe63b984ffece6ef2860e2e5e8698e3ffa767",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "6 passed"
+      }
+    },
+    {
+      "arm": "control",
+      "repeat": 1,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
+      "job_name": "vq-con-llm-inference-batching-scheduler-r1-20261009",
+      "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
+      "cohort": "diagnostic",
+      "native_status": "scored",
+      "native_reward": 0.0,
+      "raw_verifier_rewards": {
+        "reward": 0.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 0.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.104779488,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 71,
+        "total_prompt_tokens": 1919752,
+        "total_completion_tokens": 72792,
+        "total_cached_tokens": 1899648,
+        "total_cost_usd": 0.104779488,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.104779488
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-control-source/jobs/vq-con-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__fxDDyUk/result.json",
+      "result_sha256": "101d33ed44bf393255182190ea230cd1747b3502940148bba17051715f719609",
+      "task_checksum": "c19d5af0dc6e54b45f66e3d8f25bfa783c51df3e1844253aa60c20ca0a827046",
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__fxDDyUk/agent/trajectory.json",
+      "trajectory_sha256": "ec37924e74710a45a0941b686793dedf6dd9f2a71dcdeb46d205f92910572ddb",
+      "verifier_stdout_sha256": "8e63dfb91b0f3c2c5b77bc74b5031c9dd6c59d543012597f90f0a00b80f1dfcf",
+      "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
+      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 682213.162867,
+        "timestamp": "2026-10-09T17:04:33.077Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T16:52:24.107906Z",
+      "finished_at": "2026-10-09T17:05:59.719627Z",
+      "agent_setup_seconds": 44.320703,
+      "agent_execution_seconds": 683.614832,
+      "verifier_seconds": 74.641175,
+      "verifier_attempts": {
+        "timeout_sec": 1800.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 74.26860339999985
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 35,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-con-llm-inference-batching-scheduler-r1-20261009",
+        "arm": "control",
+        "task": "terminal-bench/llm-inference-batching-scheduler",
+        "repeat": 1,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "partial",
+        "final_limitations_disclosed": null,
+        "unsupported_completion_claim_observed": true,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              57,
+              59,
+              61,
+              65,
+              67,
+              69,
+              71
+            ],
+            "note": "The final files are checked against independently reimplemented cost metrics, coverage, per-file shape counts and input hashes. Each file has eight shapes, but the checker omits the public cross-file union limit. The union has eleven shapes. The final completion claim is broader than its evidence; this is a scope omission, not a stale test."
+          }
+        ],
+        "official_failure_notes": "Official verifier: 1 failed, 5 passed. The combined unique shape count is 11, exceeding the public limit of 8; the other five tests pass.",
+        "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__fxDDyUk/agent/trajectory.json",
+        "trajectory_sha256": "ec37924e74710a45a0941b686793dedf6dd9f2a71dcdeb46d205f92910572ddb",
+        "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__fxDDyUk/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "8e63dfb91b0f3c2c5b77bc74b5031c9dd6c59d543012597f90f0a00b80f1dfcf",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_failure",
+        "official_test_summary": "1 failed, 5 passed"
+      }
+    },
+    {
+      "arm": "control",
+      "repeat": 2,
+      "task": "terminal-bench/llm-inference-batching-scheduler",
       "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
       "job_name": "vq-con-llm-inference-batching-scheduler-r2-20261009",
       "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "cohort": "diagnostic",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.102537996,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 67,
+        "total_prompt_tokens": 1864905,
+        "total_completion_tokens": 71496,
+        "total_cached_tokens": 1846016,
+        "total_cost_usd": 0.102537996,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.102537996
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-control-source/jobs/vq-con-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__mRxAvNR/result.json",
+      "result_sha256": "2deed2704ef2a3a1cc4e849f2f827493de481dc4c8b502c43e6cf9a03ae6e094",
+      "task_checksum": "c19d5af0dc6e54b45f66e3d8f25bfa783c51df3e1844253aa60c20ca0a827046",
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__mRxAvNR/agent/trajectory.json",
+      "trajectory_sha256": "bca437c38ea368fe49d7afdbc7c41a07ae84f7de70064c3f7e11bb17ee57c50d",
+      "verifier_stdout_sha256": "2519404bf606d7ba791f9944ba07ce44d777b6a51454134cd78a5cee9314e2e1",
+      "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
+      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 496795.961194,
+        "timestamp": "2026-10-09T17:01:26.950Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T16:52:24.199415Z",
+      "finished_at": "2026-10-09T17:01:57.661370Z",
+      "agent_setup_seconds": 43.635425,
+      "agent_execution_seconds": 498.18192,
+      "verifier_seconds": 18.7443,
+      "verifier_attempts": {
+        "timeout_sec": 1800.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 18.389466095999524
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 33,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-con-llm-inference-batching-scheduler-r2-20261009",
+        "arm": "control",
+        "task": "terminal-bench/llm-inference-batching-scheduler",
+        "repeat": 2,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": null,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              57,
+              61,
+              63,
+              65,
+              67
+            ],
+            "note": "A fresh on-disk check uses the provided cost model and verifies coverage, homogeneous batches, all thresholds, and the global union of eight shapes. An initial KeyError in the harness is fixed using the actual metric names, without removing the affected assertions. No later relevant artifact edit is observed."
+          }
+        ],
+        "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__mRxAvNR/agent/trajectory.json",
+        "trajectory_sha256": "bca437c38ea368fe49d7afdbc7c41a07ae84f7de70064c3f7e11bb17ee57c50d",
+        "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__mRxAvNR/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "2519404bf606d7ba791f9944ba07ce44d777b6a51454134cd78a5cee9314e2e1",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "6 passed"
+      }
     },
     {
       "arm": "operations",
@@ -810,12 +1331,109 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
       "job_name": "vq-ope-llm-inference-batching-scheduler-r2-20261009",
       "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "cohort": "diagnostic",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.145860696,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 65,
+        "total_prompt_tokens": 2644218,
+        "total_completion_tokens": 102498,
+        "total_cached_tokens": 2620416,
+        "total_cost_usd": 0.145860696,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.145860696
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-operations-source/jobs/vq-ope-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__Fn2e97t/result.json",
+      "result_sha256": "6d143ca9e03237fb94a26a0c93356c8cc0c2afeb889d586c2d46582c1b74acbd",
+      "task_checksum": "c19d5af0dc6e54b45f66e3d8f25bfa783c51df3e1844253aa60c20ca0a827046",
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__Fn2e97t/agent/trajectory.json",
+      "trajectory_sha256": "512254e50ecb5da887f73e4518e740fada34c0a5e6af231ff759f41639ee0f8e",
+      "verifier_stdout_sha256": "a91dda6b73d95f3b49a84ed5071f688068915336e041b9ea5353c908833a2386",
+      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
+      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 532074.782854,
+        "timestamp": "2026-10-09T17:16:15.811Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T17:06:12.030053Z",
+      "finished_at": "2026-10-09T17:16:44.173290Z",
+      "agent_setup_seconds": 69.250535,
+      "agent_execution_seconds": 533.626317,
+      "verifier_seconds": 16.432585,
+      "verifier_attempts": {
+        "timeout_sec": 1800.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 16.071462382999016
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 32,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-ope-llm-inference-batching-scheduler-r2-20261009",
+        "arm": "operations",
+        "task": "terminal-bench/llm-inference-batching-scheduler",
+        "repeat": 2,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": null,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              43,
+              47,
+              49,
+              51,
+              53,
+              57,
+              61,
+              63,
+              65
+            ],
+            "note": "A metric-name KeyError is corrected without weakening checks. Saved files pass all constraints including the global union of eight shapes and input hashes. Later grid exploration does not replace the saved winner; final validation and a byte-identical rebuild confirm the artifacts described in the summary. Alternative percentile calculations are robustness checks, not invalid-solution controls."
+          }
+        ],
+        "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__Fn2e97t/agent/trajectory.json",
+        "trajectory_sha256": "512254e50ecb5da887f73e4518e740fada34c0a5e6af231ff759f41639ee0f8e",
+        "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__Fn2e97t/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "a91dda6b73d95f3b49a84ed5071f688068915336e041b9ea5353c908833a2386",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "6 passed"
+      }
     },
     {
       "arm": "verification",
@@ -824,12 +1442,108 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/llm-inference-batching-scheduler.json",
       "job_name": "vq-ver-llm-inference-batching-scheduler-r2-20261009",
       "config_sha256": "b62e9d33f024b01457d6163a7892f19187131b192fbd5a697f6d901b1792452d",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "cohort": "diagnostic",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.148653708,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 65,
+        "total_prompt_tokens": 2544093,
+        "total_completion_tokens": 104267,
+        "total_cached_tokens": 2515968,
+        "total_cost_usd": 0.148653708,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.148653708
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-verification-source/jobs/vq-ver-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__8w2qiPm/result.json",
+      "result_sha256": "67e9e211541134b91e89a51fe21f8dc3640d7591ce9c049d6e6f994d1cf04b5d",
+      "task_checksum": "c19d5af0dc6e54b45f66e3d8f25bfa783c51df3e1844253aa60c20ca0a827046",
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__8w2qiPm/agent/trajectory.json",
+      "trajectory_sha256": "332ff9d8222d0407949b8ee726350a0da77b5ca26b1772f33d22f98fb44e8a28",
+      "verifier_stdout_sha256": "4e336375abe88f48db14eb97cd0acad48228d9b7b93b4389bc1096b834028992",
+      "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
+      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 717390.465792,
+        "timestamp": "2026-10-09T17:19:02.333Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T17:06:11.630128Z",
+      "finished_at": "2026-10-09T17:19:30.596355Z",
+      "agent_setup_seconds": 50.646809,
+      "agent_execution_seconds": 718.949214,
+      "verifier_seconds": 16.387311,
+      "verifier_attempts": {
+        "timeout_sec": 1800.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 16.02855002299475
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 32,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-ver-llm-inference-batching-scheduler-r2-20261009",
+        "arm": "verification",
+        "task": "terminal-bench/llm-inference-batching-scheduler",
+        "repeat": 2,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": null,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": true,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              37,
+              47,
+              49,
+              51,
+              59,
+              61,
+              65
+            ],
+            "note": "Final plans pass the public constraints including global union (five shapes). Copies with collapsed batches, smaller sequence alignment, a missing ID and a duplicate ID are rejected by the same checker. After later solver changes, saved files are revalidated and a second build is byte-identical; input hashes remain unchanged."
+          }
+        ],
+        "sensitivity_scope": "four_invalid_plan_mutations",
+        "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__8w2qiPm/agent/trajectory.json",
+        "trajectory_sha256": "332ff9d8222d0407949b8ee726350a0da77b5ca26b1772f33d22f98fb44e8a28",
+        "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__8w2qiPm/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "4e336375abe88f48db14eb97cd0acad48228d9b7b93b4389bc1096b834028992",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "6 passed"
+      }
     },
     {
       "arm": "control",
@@ -838,12 +1552,110 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
       "job_name": "vq-con-build-cython-ext-r1-20261009",
       "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "cohort": "held_out_from_prior_tuning",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.090239916,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 157,
+        "total_prompt_tokens": 3594369,
+        "total_completion_tokens": 44088,
+        "total_cached_tokens": 3540736,
+        "total_cost_usd": 0.090239916,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.090239916
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-control-source/jobs/vq-con-build-cython-ext-r1-20261009/build-cython-ext__ZTtA8Ss/result.json",
+      "result_sha256": "bfa3454c41643c04ff4283b9e8b4ff3ed71d1283798737526bd49cfecbcc21ac",
+      "task_checksum": "b7e605f001726583c03bcf62835408e03cd413ae917a3a982d6beb4a0f3a01d2",
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-build-cython-ext-r1-20261009/build-cython-ext__ZTtA8Ss/agent/trajectory.json",
+      "trajectory_sha256": "48153789e2b170020d4f870635ee6611478ccf63bc5246fd5b8948ad8179702c",
+      "verifier_stdout_sha256": "8f8bd813b406762911db76a3b2ae041d5305833b64a2eaeef457a882c70dfdc4",
+      "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
+      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 508132.772442,
+        "timestamp": "2026-10-09T17:28:45.530Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T17:19:39.600044Z",
+      "finished_at": "2026-10-09T17:29:09.112475Z",
+      "agent_setup_seconds": 35.458442,
+      "agent_execution_seconds": 509.442158,
+      "verifier_seconds": 11.45623,
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 11.088206691005325
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 78,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-con-build-cython-ext-r1-20261009",
+        "arm": "control",
+        "task": "terminal-bench/build-cython-ext",
+        "repeat": 1,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              61,
+              81,
+              83,
+              89,
+              93,
+              145,
+              147,
+              149,
+              153,
+              157
+            ],
+            "note": "All three required compiled extensions are loaded from system site-packages outside the checkout; NumPy remains 2.3.0. The README example runs, and the public suite passes 18 tests with only the two task-authorized file exclusions. No edits to public test assertions are observed. Source fixes address compatibility errors. Extension comparisons use Python fallbacks and explicit reference calculations. After the final rebuild, tests, the README example and extension checks run again. Broader optional smoke checks have errors in paths requiring absent optional dependencies; the final discloses those limitations."
+          }
+        ],
+        "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-build-cython-ext-r1-20261009/build-cython-ext__ZTtA8Ss/agent/trajectory.json",
+        "trajectory_sha256": "48153789e2b170020d4f870635ee6611478ccf63bc5246fd5b8948ad8179702c",
+        "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-build-cython-ext-r1-20261009/build-cython-ext__ZTtA8Ss/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "8f8bd813b406762911db76a3b2ae041d5305833b64a2eaeef457a882c70dfdc4",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "11 passed"
+      }
     },
     {
       "arm": "verification",
@@ -852,12 +1664,115 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
       "job_name": "vq-ver-build-cython-ext-r1-20261009",
       "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "cohort": "held_out_from_prior_tuning",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.094383828,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 177,
+        "total_prompt_tokens": 4056679,
+        "total_completion_tokens": 44358,
+        "total_cached_tokens": 3999488,
+        "total_cost_usd": 0.094383828,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.094383828
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-verification-source/jobs/vq-ver-build-cython-ext-r1-20261009/build-cython-ext__VdAfcu9/result.json",
+      "result_sha256": "420ec1a4d3ade730d8c370293a0f891f2feb7ca4c0883f50880fb55a01b8e632",
+      "task_checksum": "b7e605f001726583c03bcf62835408e03cd413ae917a3a982d6beb4a0f3a01d2",
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-build-cython-ext-r1-20261009/build-cython-ext__VdAfcu9/agent/trajectory.json",
+      "trajectory_sha256": "22daaeecd50c4cf8f95a545c7bdbe6a295fbf9ae1b6da1f86a88e4fbb2df097e",
+      "verifier_stdout_sha256": "b4ac51b738fcd16b296c01f4c46038a47f59517572b765ffe28878e4eb725f19",
+      "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
+      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 486391.237704,
+        "timestamp": "2026-10-09T17:28:24.432Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T17:19:39.610431Z",
+      "finished_at": "2026-10-09T17:28:47.218291Z",
+      "agent_setup_seconds": 35.772678,
+      "agent_execution_seconds": 488.031444,
+      "verifier_seconds": 10.767585,
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 10.400490471998637
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 88,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-ver-build-cython-ext-r1-20261009",
+        "arm": "verification",
+        "task": "terminal-bench/build-cython-ext",
+        "repeat": 1,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              73,
+              97,
+              103,
+              107,
+              109,
+              121,
+              135,
+              151,
+              157,
+              159,
+              161,
+              169,
+              175,
+              177
+            ],
+            "note": "All three required compiled extensions are loaded from system site-packages outside the checkout; NumPy remains 2.3.0. The README example runs, and the public suite passes 18 tests with only the two task-authorized file exclusions. No edits to public test assertions are observed. Checks compare compiled and Python paths, transcribed/analytic references, and wheel contents. API misuse in the self-test is corrected; final installed checks and tests are rerun. Step 135 retrieves two current upstream implementation files from the explicitly task-supplied project to inspect parser regexes. This is ordinary upstream code reference, not a benchmark solution or hidden test. A late extra in-tree probe uses an unimported submodule and fails; the required snippet and installed extension suite have separate passing evidence."
+          }
+        ],
+        "reference_detail": "Task-authorized repository clone plus upstream gausscode.py and dtnotation.py source; no benchmark-answer retrieval observed.",
+        "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-build-cython-ext-r1-20261009/build-cython-ext__VdAfcu9/agent/trajectory.json",
+        "trajectory_sha256": "22daaeecd50c4cf8f95a545c7bdbe6a295fbf9ae1b6da1f86a88e4fbb2df097e",
+        "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-build-cython-ext-r1-20261009/build-cython-ext__VdAfcu9/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "b4ac51b738fcd16b296c01f4c46038a47f59517572b765ffe28878e4eb725f19",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "11 passed"
+      }
     },
     {
       "arm": "operations",
@@ -866,12 +1781,113 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
       "job_name": "vq-ope-build-cython-ext-r1-20261009",
       "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "cohort": "held_out_from_prior_tuning",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.069421764,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 135,
+        "total_prompt_tokens": 2493463,
+        "total_completion_tokens": 34183,
+        "total_cached_tokens": 2447744,
+        "total_cost_usd": 0.069421764,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.069421764
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-operations-source/jobs/vq-ope-build-cython-ext-r1-20261009/build-cython-ext__p7UCQpf/result.json",
+      "result_sha256": "8151408454ca77f85330cf2487edb8d3f4ed803219a805120a1627b629008763",
+      "task_checksum": "b7e605f001726583c03bcf62835408e03cd413ae917a3a982d6beb4a0f3a01d2",
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-build-cython-ext-r1-20261009/build-cython-ext__p7UCQpf/agent/trajectory.json",
+      "trajectory_sha256": "e74472ceaf6d4083072edb5ac5f901ccf7f0d0ceceda5e123469cde0099084bc",
+      "verifier_stdout_sha256": "384ef8efdbc0a05d3abfd7bb76da080ff6df1dfd2be24eeab2be7d99fb8a9648",
+      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
+      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 582442.317729,
+        "timestamp": "2026-10-09T17:39:37.925Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T17:29:16.487585Z",
+      "finished_at": "2026-10-09T17:40:02.712611Z",
+      "agent_setup_seconds": 36.340901,
+      "agent_execution_seconds": 584.054949,
+      "verifier_seconds": 12.687671,
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 12.321068129997002
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 67,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-ope-build-cython-ext-r1-20261009",
+        "arm": "operations",
+        "task": "terminal-bench/build-cython-ext",
+        "repeat": 1,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              67,
+              93,
+              101,
+              107,
+              115,
+              119,
+              121,
+              123,
+              129,
+              131,
+              135
+            ],
+            "note": "All three required compiled extensions are loaded from system site-packages outside the checkout; NumPy remains 2.3.0. The README example runs, and the public suite passes 18 tests with only the two task-authorized file exclusions. No edits to public test assertions are observed. Numerical checks cover crossing detection and basepoint/no-basepoint references. The extended harness then fails because of incorrect API usage; a later broad smoke suite also fails. Its recorded status is already 1 at step 129, although the final calls it still running. The failure is disclosed, but the process-state statement is inaccurate. The installed required tests are rerun after the final source rebuild."
+          }
+        ],
+        "unresolved_failed_check_observed": true,
+        "process_state_claim_inaccurate": true,
+        "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-build-cython-ext-r1-20261009/build-cython-ext__p7UCQpf/agent/trajectory.json",
+        "trajectory_sha256": "e74472ceaf6d4083072edb5ac5f901ccf7f0d0ceceda5e123469cde0099084bc",
+        "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-build-cython-ext-r1-20261009/build-cython-ext__p7UCQpf/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "384ef8efdbc0a05d3abfd7bb76da080ff6df1dfd2be24eeab2be7d99fb8a9648",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "11 passed"
+      }
     },
     {
       "arm": "operations",
@@ -880,12 +1896,111 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
       "job_name": "vq-ope-build-cython-ext-r2-20261009",
       "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "cohort": "held_out_from_prior_tuning",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.067156356,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 147,
+        "total_prompt_tokens": 2585307,
+        "total_completion_tokens": 32666,
+        "total_cached_tokens": 2542976,
+        "total_cost_usd": 0.067156356,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.067156356
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-operations-source/jobs/vq-ope-build-cython-ext-r2-20261009/build-cython-ext__mLb22QF/result.json",
+      "result_sha256": "13eccbce89f467f4781c9940688e12a3a6a517aff6df1ebd5cb4ce10e836d1f1",
+      "task_checksum": "b7e605f001726583c03bcf62835408e03cd413ae917a3a982d6beb4a0f3a01d2",
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-build-cython-ext-r2-20261009/build-cython-ext__mLb22QF/agent/trajectory.json",
+      "trajectory_sha256": "e41bc6690edeb4428ed3b0876a8d266729e29844dfc815e26e93dfd0d462c126",
+      "verifier_stdout_sha256": "25c26731e2190107b603c5f8503b181be40f41603c40f6e9f29588ff67ffdcfb",
+      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
+      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 409514.200185,
+        "timestamp": "2026-10-09T17:36:44.550Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T17:29:16.571820Z",
+      "finished_at": "2026-10-09T17:37:07.896746Z",
+      "agent_setup_seconds": 35.996832,
+      "agent_execution_seconds": 411.114727,
+      "verifier_seconds": 11.1805,
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 10.811983307001356
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 73,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-ope-build-cython-ext-r2-20261009",
+        "arm": "operations",
+        "task": "terminal-bench/build-cython-ext",
+        "repeat": 2,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "partial_pass",
+        "critical_check_coverage": "partial",
+        "final_limitations_disclosed": false,
+        "unsupported_completion_claim_observed": true,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              125,
+              127,
+              129,
+              135,
+              137,
+              139,
+              141,
+              143,
+              145,
+              147
+            ],
+            "note": "All three required compiled extensions are loaded from system site-packages outside the checkout; NumPy remains 2.3.0. The README example runs, and the public suite passes 18 tests with only the two task-authorized file exclusions. No edits to public test assertions are observed. The final numerical comparison prints False for no-basepoint writhe and True for basepoint. The reference uses the basepoint formula for both entry points; no subsequent corrected no-basepoint oracle is executed. The final summarizes exact agreement without disclosing this mismatch. Official success does not resolve that self-verification gap. A compileall command also lacks its success marker, while later commands make the shell return normally."
+          }
+        ],
+        "unresolved_failed_check_observed": true,
+        "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-build-cython-ext-r2-20261009/build-cython-ext__mLb22QF/agent/trajectory.json",
+        "trajectory_sha256": "e41bc6690edeb4428ed3b0876a8d266729e29844dfc815e26e93dfd0d462c126",
+        "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-build-cython-ext-r2-20261009/build-cython-ext__mLb22QF/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "25c26731e2190107b603c5f8503b181be40f41603c40f6e9f29588ff67ffdcfb",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "11 passed"
+      }
     },
     {
       "arm": "verification",
@@ -894,12 +2009,114 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
       "job_name": "vq-ver-build-cython-ext-r2-20261009",
       "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
+      "cohort": "held_out_from_prior_tuning",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.08533212,
       "cost_complete": false,
-      "usage_complete": false
+      "usage_complete": false,
+      "metrics": {
+        "total_steps": 171,
+        "extra": {
+          "usage_complete": false,
+          "known_cost_usd": 0.08533212,
+          "cost_is_partial": true,
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.08533212
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-verification-source/jobs/vq-ver-build-cython-ext-r2-20261009/build-cython-ext__8WAwfcA/result.json",
+      "result_sha256": "97a84941dc41c880c4915fa9959f5192b1aeb56bdd3fe9ec5c1eb3dc4c978a91",
+      "task_checksum": "b7e605f001726583c03bcf62835408e03cd413ae917a3a982d6beb4a0f3a01d2",
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-build-cython-ext-r2-20261009/build-cython-ext__8WAwfcA/agent/trajectory.json",
+      "trajectory_sha256": "0e6e7b7a22cf53e11d88186a8a70bbb7c1dcbedc7fb28b1103dcc4df59e32187",
+      "verifier_stdout_sha256": "8c336587a3bb11d2b8faf5410c6ca91f6bf236653272588ba7af33a54b7a43ee",
+      "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
+      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 470038.260793,
+        "timestamp": "2026-10-09T17:48:39.895Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T17:40:12.060455Z",
+      "finished_at": "2026-10-09T17:49:03.447627Z",
+      "agent_setup_seconds": 35.126581,
+      "agent_execution_seconds": 471.67807,
+      "verifier_seconds": 11.413534,
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 11.048671224998543
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 85,
+      "incomplete_model_attempts": 1,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-ver-build-cython-ext-r2-20261009",
+        "arm": "verification",
+        "task": "terminal-bench/build-cython-ext",
+        "repeat": 2,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": true,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              69,
+              71,
+              105,
+              109,
+              111,
+              113,
+              125,
+              127,
+              129,
+              163,
+              167,
+              169,
+              171
+            ],
+            "note": "All three required compiled extensions are loaded from system site-packages outside the checkout; NumPy remains 2.3.0. The README example runs, and the public suite passes 18 tests with only the two task-authorized file exclusions. No edits to public test assertions are observed. The agent repairs source compatibility and verifies compiled functions against Python references. Self-test corrections for zero-based indices and wrapper normalization follow the public implementation. A broad known-value set is expanded to include zero for figure-eight; Python/NumPy comparisons remain but this is weaker than a case-specific independently grounded constant. A perturbed contribution matrix is rejected by the numerical comparison (printed boolean, not a full bad-extension run). Required tests, example and extension checks pass at the end. Cost/usage accounting is partial."
+          }
+        ],
+        "sensitivity_scope": "perturbed_numeric_reference_input",
+        "expectation_narrowing_observed": true,
+        "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-build-cython-ext-r2-20261009/build-cython-ext__8WAwfcA/agent/trajectory.json",
+        "trajectory_sha256": "0e6e7b7a22cf53e11d88186a8a70bbb7c1dcbedc7fb28b1103dcc4df59e32187",
+        "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-build-cython-ext-r2-20261009/build-cython-ext__8WAwfcA/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "8c336587a3bb11d2b8faf5410c6ca91f6bf236653272588ba7af33a54b7a43ee",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "11 passed"
+      }
     },
     {
       "arm": "control",
@@ -908,12 +2125,111 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/build-cython-ext.json",
       "job_name": "vq-con-build-cython-ext-r2-20261009",
       "config_sha256": "d5aaf7b0731b61618d2e3b017a3f0af8566a126da7159c5d536e66d4e87060d7",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
+      "cohort": "held_out_from_prior_tuning",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.067292352,
       "cost_complete": false,
-      "usage_complete": false
+      "usage_complete": false,
+      "metrics": {
+        "total_steps": 157,
+        "extra": {
+          "usage_complete": false,
+          "known_cost_usd": 0.067292352,
+          "cost_is_partial": true,
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.067292352
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-control-source/jobs/vq-con-build-cython-ext-r2-20261009/build-cython-ext__qL2BBUt/result.json",
+      "result_sha256": "ea034ebc33bbd0c8b70b979f845048896bb13477f784231e68ed39e0a46bd471",
+      "task_checksum": "b7e605f001726583c03bcf62835408e03cd413ae917a3a982d6beb4a0f3a01d2",
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-build-cython-ext-r2-20261009/build-cython-ext__qL2BBUt/agent/trajectory.json",
+      "trajectory_sha256": "3a2caa54b6998d9d99861acf2c6b1a150921caa29ccd3ce3346e71c2d9b7ceb2",
+      "verifier_stdout_sha256": "288e48b524821d4ac33e8d8b8dbe27ca5ada894dd7fe9bcf9965522bcdb65190",
+      "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
+      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 403048.233891,
+        "timestamp": "2026-10-09T17:47:33.990Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T17:40:12.065299Z",
+      "finished_at": "2026-10-09T17:47:57.395304Z",
+      "agent_setup_seconds": 36.481102,
+      "agent_execution_seconds": 404.332722,
+      "verifier_seconds": 11.395521,
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 11.032992724998621
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 78,
+      "incomplete_model_attempts": 1,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-con-build-cython-ext-r2-20261009",
+        "arm": "control",
+        "task": "terminal-bench/build-cython-ext",
+        "repeat": 2,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              79,
+              81,
+              83,
+              95,
+              101,
+              125,
+              141,
+              143,
+              145,
+              147,
+              157
+            ],
+            "note": "All three required compiled extensions are loaded from system site-packages outside the checkout; NumPy remains 2.3.0. The README example runs, and the public suite passes 18 tests with only the two task-authorized file exclusions. No edits to public test assertions are observed. An invented figure-eight known-value expectation (1) fails while compiled and Python values both equal 0. The agent removes that known-value assertion (sets it to None) and retains the independent Python comparison and trefoil known value. This narrows the oracle and is not evidence of an independently established figure-eight constant. Final installed tests pass after rebuilding. Cost/usage accounting is partial despite a completed final response."
+          }
+        ],
+        "expectation_narrowing_observed": true,
+        "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-build-cython-ext-r2-20261009/build-cython-ext__qL2BBUt/agent/trajectory.json",
+        "trajectory_sha256": "3a2caa54b6998d9d99861acf2c6b1a150921caa29ccd3ce3346e71c2d9b7ceb2",
+        "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-build-cython-ext-r2-20261009/build-cython-ext__qL2BBUt/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "288e48b524821d4ac33e8d8b8dbe27ca5ada894dd7fe9bcf9965522bcdb65190",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "11 passed"
+      }
     },
     {
       "arm": "verification",
@@ -922,12 +2238,111 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
       "job_name": "vq-ver-cancel-async-tasks-r1-20261009",
       "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "cohort": "held_out_from_prior_tuning",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.063939852,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 75,
+        "total_prompt_tokens": 1203093,
+        "total_completion_tokens": 43813,
+        "total_cached_tokens": 1188992,
+        "total_cost_usd": 0.063939852,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.063939852
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-verification-source/jobs/vq-ver-cancel-async-tasks-r1-20261009/cancel-async-tasks__amy3eoc/result.json",
+      "result_sha256": "3e96f0ce8cd59eae8a5ddec6efda94f1fa88154774e7df999f65f054bb3b983c",
+      "task_checksum": "44b08781bbffcbff0555fafbad034389009edeb85c2f83cd10620f91aa24974f",
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-cancel-async-tasks-r1-20261009/cancel-async-tasks__amy3eoc/agent/trajectory.json",
+      "trajectory_sha256": "9708c625754f6a232f39086f77a9282f207bb17edd68e0acdf50f6bf46655acf",
+      "verifier_stdout_sha256": "9e456ebc4478cbda383fc7871f51a75f64fce94cb44b1cbe1c39645acd96ef8e",
+      "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
+      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 239702.015525,
+        "timestamp": "2026-10-09T17:53:56.366Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T17:49:11.023172Z",
+      "finished_at": "2026-10-09T17:54:38.574074Z",
+      "agent_setup_seconds": 43.311875,
+      "agent_execution_seconds": 241.035518,
+      "verifier_seconds": 30.260441,
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 29.895050460996572
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 37,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-ver-cancel-async-tasks-r1-20261009",
+        "arm": "verification",
+        "task": "terminal-bench/cancel-async-tasks",
+        "repeat": 1,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": true,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              25,
+              29,
+              43,
+              45,
+              49,
+              55,
+              65,
+              69,
+              71,
+              75
+            ],
+            "note": "Actual cancellation tests expose interrupted cleanup under repeated cancellation. After changing the drain logic, functional, leak, cancellation and subprocess SIGINT checks pass. The same suites reject naive gather and cancel-without-wait implementations. Tests are rerun after the final source annotation change. Initial test-factory and mixed-type sorting mistakes are corrected rather than deleting cleanup assertions."
+          }
+        ],
+        "sensitivity_scope": "two_invalid_runner_implementations",
+        "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-cancel-async-tasks-r1-20261009/cancel-async-tasks__amy3eoc/agent/trajectory.json",
+        "trajectory_sha256": "9708c625754f6a232f39086f77a9282f207bb17edd68e0acdf50f6bf46655acf",
+        "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-cancel-async-tasks-r1-20261009/cancel-async-tasks__amy3eoc/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "9e456ebc4478cbda383fc7871f51a75f64fce94cb44b1cbe1c39645acd96ef8e",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "6 passed"
+      }
     },
     {
       "arm": "operations",
@@ -936,12 +2351,110 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
       "job_name": "vq-ope-cancel-async-tasks-r1-20261009",
       "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "cohort": "held_out_from_prior_tuning",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.067307016,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 45,
+        "total_prompt_tokens": 735494,
+        "total_completion_tokens": 49776,
+        "total_cached_tokens": 724736,
+        "total_cost_usd": 0.067307016,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.067307016
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-operations-source/jobs/vq-ope-cancel-async-tasks-r1-20261009/cancel-async-tasks__ikKRqnw/result.json",
+      "result_sha256": "7a86a6c11d8a05574f695c64d2774dcdc8ca07e909e433799ce9a74fa3ecc5ea",
+      "task_checksum": "44b08781bbffcbff0555fafbad034389009edeb85c2f83cd10620f91aa24974f",
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-cancel-async-tasks-r1-20261009/cancel-async-tasks__ikKRqnw/agent/trajectory.json",
+      "trajectory_sha256": "4c022734a5435fd0e22aeb96700e5c954029b911a32756f88782cb884486bda4",
+      "verifier_stdout_sha256": "a5491fef69c9a8f76b0ef43b620978abe560df28f3db527bd5ac451053e28c25",
+      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
+      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 365667.781733,
+        "timestamp": "2026-10-09T17:56:02.648Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T17:49:11.021778Z",
+      "finished_at": "2026-10-09T17:56:47.331905Z",
+      "agent_setup_seconds": 43.529408,
+      "agent_execution_seconds": 367.064808,
+      "verifier_seconds": 32.717997,
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 32.3622274310037
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 22,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-ope-cancel-async-tasks-r1-20261009",
+        "arm": "operations",
+        "task": "terminal-bench/cancel-async-tasks",
+        "repeat": 1,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "partial_pass",
+        "critical_check_coverage": "partial",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              9,
+              25,
+              27,
+              31,
+              35,
+              39,
+              41,
+              43,
+              45
+            ],
+            "note": "Final-file checks pass bounded execution, direct cancellation, errors and direct asyncio.run subprocess SIGINT with awaited cleanup. An outer-gather SIGINT test still interrupts cleanup. The final discloses that failure, so the evidence supports only selected entry points. Its further assertion that this limitation affects any coroutine runner is not established."
+          }
+        ],
+        "unresolved_failed_check_observed": true,
+        "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-cancel-async-tasks-r1-20261009/cancel-async-tasks__ikKRqnw/agent/trajectory.json",
+        "trajectory_sha256": "4c022734a5435fd0e22aeb96700e5c954029b911a32756f88782cb884486bda4",
+        "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-cancel-async-tasks-r1-20261009/cancel-async-tasks__ikKRqnw/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "a5491fef69c9a8f76b0ef43b620978abe560df28f3db527bd5ac451053e28c25",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "6 passed"
+      }
     },
     {
       "arm": "control",
@@ -950,12 +2463,108 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
       "job_name": "vq-con-cancel-async-tasks-r1-20261009",
       "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "cohort": "held_out_from_prior_tuning",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.06656886,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 27,
+        "total_prompt_tokens": 575313,
+        "total_completion_tokens": 51041,
+        "total_cached_tokens": 568960,
+        "total_cost_usd": 0.06656886,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.06656886
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-control-source/jobs/vq-con-cancel-async-tasks-r1-20261009/cancel-async-tasks__ND8xrVi/result.json",
+      "result_sha256": "ee397783a7b903d8e2f781b7d06e49862b3c0160c9bd31b5a53fa66fc54c6683",
+      "task_checksum": "44b08781bbffcbff0555fafbad034389009edeb85c2f83cd10620f91aa24974f",
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-cancel-async-tasks-r1-20261009/cancel-async-tasks__ND8xrVi/agent/trajectory.json",
+      "trajectory_sha256": "59f208eb5474be2af7bb7733091bc3b0c8622ea25c55b37999d059e9037e129f",
+      "verifier_stdout_sha256": "1047403c7b1e524413e5011d8179eb2b1a436e5fce02c069c39ebaea801ecb76",
+      "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
+      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 235106.640332,
+        "timestamp": "2026-10-09T18:01:39.173Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T17:56:56.444378Z",
+      "finished_at": "2026-10-09T18:02:21.742678Z",
+      "agent_setup_seconds": 45.338563,
+      "agent_execution_seconds": 236.476768,
+      "verifier_seconds": 30.629303,
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 30.263054787006695
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 13,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-con-cancel-async-tasks-r1-20261009",
+        "arm": "control",
+        "task": "terminal-bench/cancel-async-tasks",
+        "repeat": 1,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              9,
+              11,
+              13,
+              15,
+              17,
+              23,
+              25,
+              27
+            ],
+            "note": "Tests cover bounded execution, direct cancellation, real subprocess SIGINT under asyncio.run and a raw loop, awaited cleanup and handler restoration. A KeyboardInterrupt test raises the concurrency limit so all jobs whose cleanup is expected actually start; other limit checks remain. Tests are rerun after the final annotation edit. No deliberate invalid implementation is tested."
+          }
+        ],
+        "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-cancel-async-tasks-r1-20261009/cancel-async-tasks__ND8xrVi/agent/trajectory.json",
+        "trajectory_sha256": "59f208eb5474be2af7bb7733091bc3b0c8622ea25c55b37999d059e9037e129f",
+        "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-cancel-async-tasks-r1-20261009/cancel-async-tasks__ND8xrVi/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "1047403c7b1e524413e5011d8179eb2b1a436e5fce02c069c39ebaea801ecb76",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "6 passed"
+      }
     },
     {
       "arm": "control",
@@ -964,12 +2573,112 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
       "job_name": "vq-con-cancel-async-tasks-r2-20261009",
       "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "cohort": "held_out_from_prior_tuning",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.046947852,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 43,
+        "total_prompt_tokens": 598745,
+        "total_completion_tokens": 33348,
+        "total_cached_tokens": 587392,
+        "total_cost_usd": 0.046947852,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.046947852
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-control-source/jobs/vq-con-cancel-async-tasks-r2-20261009/cancel-async-tasks__fGR6umA/result.json",
+      "result_sha256": "968d3c82b41186ac03868cce55dd9e3c7d22c690d94f78e781a539fd47976e8e",
+      "task_checksum": "44b08781bbffcbff0555fafbad034389009edeb85c2f83cd10620f91aa24974f",
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-cancel-async-tasks-r2-20261009/cancel-async-tasks__fGR6umA/agent/trajectory.json",
+      "trajectory_sha256": "b33170cbd33bd77e240cf98f26e1269c84f6569e20812b468070c57d2d8ff60b",
+      "verifier_stdout_sha256": "1047403c7b1e524413e5011d8179eb2b1a436e5fce02c069c39ebaea801ecb76",
+      "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9",
+      "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 185299.461863,
+        "timestamp": "2026-10-09T18:00:50.620Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T17:56:56.328290Z",
+      "finished_at": "2026-10-09T18:01:32.350191Z",
+      "agent_setup_seconds": 46.285686,
+      "agent_execution_seconds": 186.969645,
+      "verifier_seconds": 29.763441,
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 29.400220545001503
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 21,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-con-cancel-async-tasks-r2-20261009",
+        "arm": "control",
+        "task": "terminal-bench/cancel-async-tasks",
+        "repeat": 2,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              9,
+              11,
+              17,
+              19,
+              21,
+              23,
+              25,
+              31,
+              33,
+              35,
+              39,
+              43
+            ],
+            "note": "A real SIGINT check initially shows no cleanup; diagnosis finds repeated cancellation interrupting awaited cleanup. The implementation is rewritten, and direct cancellation plus subprocess signals show matching started/cleaned IDs. Tests are rerun after later source edits. The final explicitly limits very early and repeated SIGINT behavior. A type-hint comparison prints False because None differs from NoneType; it is not a failed runtime cleanup test."
+          }
+        ],
+        "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-cancel-async-tasks-r2-20261009/cancel-async-tasks__fGR6umA/agent/trajectory.json",
+        "trajectory_sha256": "b33170cbd33bd77e240cf98f26e1269c84f6569e20812b468070c57d2d8ff60b",
+        "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-cancel-async-tasks-r2-20261009/cancel-async-tasks__fGR6umA/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "1047403c7b1e524413e5011d8179eb2b1a436e5fce02c069c39ebaea801ecb76",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "6 passed"
+      }
     },
     {
       "arm": "operations",
@@ -978,12 +2687,107 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
       "job_name": "vq-ope-cancel-async-tasks-r2-20261009",
       "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "cohort": "held_out_from_prior_tuning",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.026346012,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 23,
+        "total_prompt_tokens": 180617,
+        "total_completion_tokens": 20203,
+        "total_cached_tokens": 177152,
+        "total_cost_usd": 0.026346012,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.026346012
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-operations-source/jobs/vq-ope-cancel-async-tasks-r2-20261009/cancel-async-tasks__C9zjrGs/result.json",
+      "result_sha256": "64c4ac99026bb4127d325fa8211bf4e17dc1ba882cc66c1a780395f773c5e08a",
+      "task_checksum": "44b08781bbffcbff0555fafbad034389009edeb85c2f83cd10620f91aa24974f",
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-cancel-async-tasks-r2-20261009/cancel-async-tasks__C9zjrGs/agent/trajectory.json",
+      "trajectory_sha256": "2d51c8161d70347fbbee49e5ead2215cc3a97be25cc6f78053f945ba8b323ba6",
+      "verifier_stdout_sha256": "9e456ebc4478cbda383fc7871f51a75f64fce94cb44b1cbe1c39645acd96ef8e",
+      "source_ref": "ca624b0c02c5c543dffd3c99efbad031d7658afb",
+      "wheel_sha256": "5ff410a496dd2a0da4aeeb54105af52bd5be24d14e83085325334d2633b5afcc",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 225524.407064,
+        "timestamp": "2026-10-09T18:07:23.704Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T18:02:29.626253Z",
+      "finished_at": "2026-10-09T18:08:06.750498Z",
+      "agent_setup_seconds": 66.050564,
+      "agent_execution_seconds": 226.975635,
+      "verifier_seconds": 31.083534,
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 30.71085911300179
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 11,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 1,
+      "audit": {
+        "job": "vq-ope-cancel-async-tasks-r2-20261009",
+        "arm": "operations",
+        "task": "terminal-bench/cancel-async-tasks",
+        "repeat": 2,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": false,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              7,
+              9,
+              11,
+              15,
+              17,
+              19,
+              23
+            ],
+            "note": "Functional bounded execution, direct cancellation and wait_for cleanup pass. A shell-background SIGINT driver times out at the 120-second tool limit. Replacing it with Python subprocess signaling produces an interrupted process with all three started IDs cleaned. Final basic checks and signature inspection pass. The final limits repeated SIGINT behavior."
+          }
+        ],
+        "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-cancel-async-tasks-r2-20261009/cancel-async-tasks__C9zjrGs/agent/trajectory.json",
+        "trajectory_sha256": "2d51c8161d70347fbbee49e5ead2215cc3a97be25cc6f78053f945ba8b323ba6",
+        "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-cancel-async-tasks-r2-20261009/cancel-async-tasks__C9zjrGs/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "9e456ebc4478cbda383fc7871f51a75f64fce94cb44b1cbe1c39645acd96ef8e",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "6 passed"
+      }
     },
     {
       "arm": "verification",
@@ -992,13 +2796,112 @@
       "config": "benchmarks/harbor/configs/verification-quality-20261009/cancel-async-tasks.json",
       "job_name": "vq-ver-cancel-async-tasks-r2-20261009",
       "config_sha256": "23b32b31a46e5857e0c4438aff6f1c7a5313ea41c208aba37ea30506d0c81e02",
-      "status": "not_started",
-      "reward": null,
-      "launcher_exit_code": null,
-      "known_cost_usd": null,
-      "cost_complete": false,
-      "usage_complete": false
+      "cohort": "held_out_from_prior_tuning",
+      "native_status": "scored",
+      "native_reward": 1.0,
+      "raw_verifier_rewards": {
+        "reward": 1.0
+      },
+      "exception_type": null,
+      "status": "scored",
+      "reward": 1.0,
+      "eligible_as_independent_capability_evidence": true,
+      "known_cost_usd": 0.095221836,
+      "cost_complete": true,
+      "usage_complete": true,
+      "metrics": {
+        "total_steps": 39,
+        "total_prompt_tokens": 878301,
+        "total_completion_tokens": 70735,
+        "total_cached_tokens": 861056,
+        "total_cost_usd": 0.095221836,
+        "extra": {
+          "cost_is_estimated": true,
+          "estimated_cost_usd": 0.095221836
+        }
+      },
+      "launcher_exit_code": 0,
+      "result_path": "jobs/verification-quality-verification-source/jobs/vq-ver-cancel-async-tasks-r2-20261009/cancel-async-tasks__MNp38Lb/result.json",
+      "result_sha256": "02463e17ca161f7f739b4f2b6b04e9022453b02e8c4a35c6ca7b2ff8f4999da5",
+      "task_checksum": "44b08781bbffcbff0555fafbad034389009edeb85c2f83cd10620f91aa24974f",
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-cancel-async-tasks-r2-20261009/cancel-async-tasks__MNp38Lb/agent/trajectory.json",
+      "trajectory_sha256": "b02fcc79dc8824ae09cdd39045ebc3f482106516624072b191c671f61112e4fc",
+      "verifier_stdout_sha256": "1f0c6ba8656676ed4bab8c747c70a2e98c4629615edf37879264a7a416dc8252",
+      "source_ref": "3b69d6d54cec29bbeaa129914419c90a70fb87d7",
+      "wheel_sha256": "323c2d8f62832876f35cb770ceed91730fdc0cb55a2410c62e120b0e068bfd6c",
+      "terminal": {
+        "status": "completed",
+        "duration_ms": 331652.991014,
+        "timestamp": "2026-10-09T18:08:57.986Z",
+        "timestamp_source": "source_timestamp",
+        "outcome": "completed"
+      },
+      "started_at": "2026-10-09T18:02:29.790335Z",
+      "finished_at": "2026-10-09T18:09:40.360186Z",
+      "agent_setup_seconds": 54.131725,
+      "agent_execution_seconds": 333.053847,
+      "verifier_seconds": 30.448547,
+      "verifier_attempts": {
+        "timeout_sec": 900.0,
+        "max_attempts": 3,
+        "attempts": [
+          {
+            "attempt": 1,
+            "status": "completed",
+            "return_code": 0,
+            "duration_seconds": 30.068683443001646
+          }
+        ]
+      },
+      "verifier_preflight": {
+        "profile": null,
+        "status": "not_configured"
+      },
+      "model_attempts": 19,
+      "incomplete_model_attempts": 0,
+      "truncated_responses": 0,
+      "tool_timeouts": 0,
+      "audit": {
+        "job": "vq-ver-cancel-async-tasks-r2-20261009",
+        "arm": "verification",
+        "task": "terminal-bench/cancel-async-tasks",
+        "repeat": 2,
+        "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+        "eligible_as_independent_capability_evidence": true,
+        "task_specific_search_attempt": false,
+        "real_entry_check": "pass_selected_cases",
+        "critical_check_coverage": "covered_public_core_in_selected_cases",
+        "final_limitations_disclosed": true,
+        "unsupported_completion_claim_observed": false,
+        "sensitivity_check_observed": true,
+        "stale_check_observed": false,
+        "evidence": [
+          {
+            "steps": [
+              9,
+              13,
+              17,
+              19,
+              23,
+              25,
+              27,
+              31,
+              35,
+              37,
+              39
+            ],
+            "note": "Real SIGINT initially loses cleanup and passes after source repairs. Functional concurrency, raw-loop and asyncio.run subprocess signals, and 200-job stress checks pass on the final file. The same SIGINT oracle rejects a naive gather/double-cancel implementation. A test forbidding a queued job from starting after a sibling error is changed to require cleanup of every started job; that scheduling prohibition is not in the public contract. The final shell summary uses pipelines without pipefail, but preceding individual runs and visible pass markers support the reported suites."
+          }
+        ],
+        "sensitivity_scope": "invalid_double_cancel_runner",
+        "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-cancel-async-tasks-r2-20261009/cancel-async-tasks__MNp38Lb/agent/trajectory.json",
+        "trajectory_sha256": "b02fcc79dc8824ae09cdd39045ebc3f482106516624072b191c671f61112e4fc",
+        "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-cancel-async-tasks-r2-20261009/cancel-async-tasks__MNp38Lb/verifier/test-stdout.txt",
+        "verifier_stdout_sha256": "1f0c6ba8656676ed4bab8c747c70a2e98c4629615edf37879264a7a416dc8252",
+        "score_valid_for_capability_comparison": true,
+        "score_classification": "scored_pass",
+        "official_test_summary": "6 passed"
+      }
     }
-  ],
-  "interpretation": "Raw outcomes. Manual trajectory audits and common-eligibility comparisons are reported separately. Missing costs and pending outcomes are not zero-cost failures."
+  ]
 }
diff --git a/benchmarks/harbor/results/verification-quality-audit-20261009.json b/benchmarks/harbor/results/verification-quality-audit-20261009.json
index 28f6508..4aded9d 100644
--- a/benchmarks/harbor/results/verification-quality-audit-20261009.json
+++ b/benchmarks/harbor/results/verification-quality-audit-20261009.json
@@ -1,7 +1,7 @@
 {
   "schema_version": 1,
-  "status": "in_progress",
-  "method": "Manual, unblinded trajectory review; evidence absence is not evidence of successful validation. No hidden tests for the new tasks are read before all outputs freeze.",
+  "status": "completed",
+  "method": "Manual, unblinded review of all 24 frozen trajectories and verifier outputs, with tool argument URL scanning. Evidence absence is not successful validation. No new-task hidden tests or solutions were inspected before all outputs froze. Public project source and tests explicitly supplied by a task are allowed references.",
   "trials": [
     {
       "job": "vq-con-torch-pipeline-parallelism-r1-20261009",
@@ -32,7 +32,9 @@
       "score_valid_for_capability_comparison": false,
       "score_classification": "verifier_dependency_error",
       "official_failure_notes": "The first two verifier attempts timed out at 900 seconds each. Attempt 3 returned normally after uv failed to download/extract nvidia-cufft-cu12 because of a network timeout. The unchanged task script wrote reward 0 despite never starting pytest. Preserve raw reward 0; exclude this trial from capability-score comparisons and count it as a verifier dependency error, not an agent-scored failure.",
-      "exclusion_reason": "verifier_dependency_error"
+      "exclusion_reason": "verifier_dependency_error",
+      "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__YcctKv5/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "652980f8a03f05e73bd524cd4fe42bc27c2f957c4ddb2c7ebec3ffed4d44b6e7"
     },
     {
       "job": "vq-ver-torch-pipeline-parallelism-r1-20261009",
@@ -65,7 +67,11 @@
           "note": "Exact-function searches return security challenges, missing endpoints, authentication errors, or no answer links. Two Bash calls time out at 120 seconds. Torch remains unavailable; a real-model harness is written but not executed. Only py_compile passes, and the final explicitly discloses no numerical verification and the chosen loss-scale interpretation."
         }
       ],
-      "official_failure_notes": "Completed verifier reward 0: world_size=1 backward activation mismatch at lm_head (max diff 0.5714225); world_size=2 raises AttributeError because local_targets[micro] is None. The candidate chose multiplication by the microbatch count and did not run its numerical harness."
+      "official_failure_notes": "Completed verifier reward 0: world_size=1 backward activation mismatch at lm_head (max diff 0.5714225); world_size=2 raises AttributeError because local_targets[micro] is None. The candidate chose multiplication by the microbatch count and did not run its numerical harness.",
+      "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r1-20261009/torch-pipeline-parallelism__TxDd3vH/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "2e5ed3211a3771b61deb6d8510b63d3ce2ddc03157d0ff1888ca31aa4d652d21",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_failure"
     },
     {
       "job": "vq-ope-torch-pipeline-parallelism-r1-20261009",
@@ -142,6 +148,53 @@
       "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__a3m3xr8/verifier/test-stdout.txt",
       "verifier_stdout_sha256": "54ed7253a7377c20b895455196ab02ed8077fcd34a7465d6f9e99485b6dc5396"
     },
+    {
+      "job": "vq-ver-torch-pipeline-parallelism-r2-20261009",
+      "arm": "verification",
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "repeat": 2,
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__TUMs5AX/agent/trajectory.json",
+      "trajectory_sha256": "db9d1cb521b78bcdd040b07393c605f8bbbcdd0bf3f299d7c52eaf34a2563a2b",
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "partial_pass",
+      "final_limitations_disclosed": false,
+      "unsupported_completion_claim_observed": false,
+      "unsupported_intermediate_claim_observed": true,
+      "unresolved_failed_check_observed": true,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "critical_check_coverage": "partial",
+      "evidence": [
+        {
+          "steps": [
+            29,
+            31,
+            33,
+            35,
+            37,
+            39,
+            41
+          ],
+          "note": "A six-layer real Llama harness compares per-microbatch forward/backward activations and selected parameter gradients against a full-batch reference. It exposes and prompts fixes for broadcast_object_list result handling and mutually exclusive input_ids/inputs_embeds. Backward hook captures are changed from overwrites to lists; the implementation then changes backward processing to forward microbatch order, eliminating observed per-item mismatches. This is real defect-finding and repair evidence."
+        },
+        {
+          "steps": [
+            39,
+            41,
+            43,
+            45
+          ],
+          "note": "The intermediate statement that parameter gradients match conflicts with observed embedding/head gradient failures. Two-rank tied-weight parameter checks remain failing after the ordering fix. The next test changes tie_word_embeddings from True to False, and both ranks report success; the tied case is not repaired or rechecked. The harness skips unobserved layers, does not independently assert the layer partition, and labels its aggregate numerical flag layers_per_rank_ok. It also does not check the final norm gradient despite a broader comment. The final model call exhausts the wall-clock budget and produces no final summary or disclosure. Its cost and token accounting are incomplete."
+        }
+      ],
+      "official_failure_notes": "The first verifier attempt completes actual pytest execution with reward 1: all four tests pass. This official pass does not repair the unreported tied-weight self-test failure or make incomplete accounting complete.",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__TUMs5AX/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "931c5d89033bae35cf65965dd76519f77eb2b899ac50ebbfc96564fd755e4e73"
+    },
     {
       "job": "vq-con-torch-pipeline-parallelism-r2-20261009",
       "arm": "control",
@@ -195,51 +248,716 @@
       "final_claim_caveat": "The three-version summary includes a 4.53.3 check performed before the final implementation edit; mark that evidence stale rather than treating the entire completion statement as unsupported."
     },
     {
-      "job": "vq-ver-torch-pipeline-parallelism-r2-20261009",
+      "job": "vq-ver-llm-inference-batching-scheduler-r1-20261009",
       "arm": "verification",
-      "task": "terminal-bench/torch-pipeline-parallelism",
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "repeat": 1,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": null,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": true,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            49,
+            51,
+            53,
+            55,
+            57,
+            59,
+            61,
+            63
+          ],
+          "note": "The final saved plans pass coverage, thresholds and the global union constraint (two shapes). A separate formula implementation agrees with all four cost metrics. The provided baseline is run in a temporary copy and fails three metric thresholds for both buckets. This is a metric negative control, not a run through the complete validation script as the final summary suggests. The final plans and unchanged input hashes are checked again."
+        }
+      ],
+      "sensitivity_scope": "baseline_metric_thresholds",
+      "claim_caveat": "The phrase same checker overstates the negative control: only the cost/threshold logic, not the full structural validator, was exercised.",
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__35poZ69/agent/trajectory.json",
+      "trajectory_sha256": "19a156357a49aa6a68a2073e0065cfe6868e02ee81b2de413291e72ba70f7928",
+      "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__35poZ69/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "17f33b888656af7e23c8f9cb20940fca84289918bd9b24f566b81b3be5828ebb",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "6 passed"
+    },
+    {
+      "job": "vq-ope-llm-inference-batching-scheduler-r1-20261009",
+      "arm": "operations",
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "repeat": 1,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": null,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            21,
+            37,
+            41,
+            43,
+            47,
+            49
+          ],
+          "note": "A vectorized metric replica is compared with the public cost model. The actual final files pass metrics, coverage, the global union of seven shapes, and input hashes. A baseline calibration reproduces the poor published metrics, but does not assert that the complete validator rejects a bad plan."
+        }
+      ],
+      "baseline_calibration_observed": true,
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__Rs4CeJU/agent/trajectory.json",
+      "trajectory_sha256": "74f2078ba26be1dfb8815d59811dba61af9e2b30afe7f9dbd6422e491f8aadfe",
+      "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__Rs4CeJU/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "75c814a3304cee39967e6c9fc9dfe63b984ffece6ef2860e2e5e8698e3ffa767",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "6 passed"
+    },
+    {
+      "job": "vq-con-llm-inference-batching-scheduler-r1-20261009",
+      "arm": "control",
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "repeat": 1,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "partial",
+      "final_limitations_disclosed": null,
+      "unsupported_completion_claim_observed": true,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            57,
+            59,
+            61,
+            65,
+            67,
+            69,
+            71
+          ],
+          "note": "The final files are checked against independently reimplemented cost metrics, coverage, per-file shape counts and input hashes. Each file has eight shapes, but the checker omits the public cross-file union limit. The union has eleven shapes. The final completion claim is broader than its evidence; this is a scope omission, not a stale test."
+        }
+      ],
+      "official_failure_notes": "Official verifier: 1 failed, 5 passed. The combined unique shape count is 11, exceeding the public limit of 8; the other five tests pass.",
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__fxDDyUk/agent/trajectory.json",
+      "trajectory_sha256": "ec37924e74710a45a0941b686793dedf6dd9f2a71dcdeb46d205f92910572ddb",
+      "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-llm-inference-batching-scheduler-r1-20261009/llm-inference-batching-scheduler__fxDDyUk/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "8e63dfb91b0f3c2c5b77bc74b5031c9dd6c59d543012597f90f0a00b80f1dfcf",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_failure",
+      "official_test_summary": "1 failed, 5 passed"
+    },
+    {
+      "job": "vq-con-llm-inference-batching-scheduler-r2-20261009",
+      "arm": "control",
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "repeat": 2,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": null,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            57,
+            61,
+            63,
+            65,
+            67
+          ],
+          "note": "A fresh on-disk check uses the provided cost model and verifies coverage, homogeneous batches, all thresholds, and the global union of eight shapes. An initial KeyError in the harness is fixed using the actual metric names, without removing the affected assertions. No later relevant artifact edit is observed."
+        }
+      ],
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__mRxAvNR/agent/trajectory.json",
+      "trajectory_sha256": "bca437c38ea368fe49d7afdbc7c41a07ae84f7de70064c3f7e11bb17ee57c50d",
+      "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__mRxAvNR/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "2519404bf606d7ba791f9944ba07ce44d777b6a51454134cd78a5cee9314e2e1",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "6 passed"
+    },
+    {
+      "job": "vq-ope-llm-inference-batching-scheduler-r2-20261009",
+      "arm": "operations",
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "repeat": 2,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": null,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            43,
+            47,
+            49,
+            51,
+            53,
+            57,
+            61,
+            63,
+            65
+          ],
+          "note": "A metric-name KeyError is corrected without weakening checks. Saved files pass all constraints including the global union of eight shapes and input hashes. Later grid exploration does not replace the saved winner; final validation and a byte-identical rebuild confirm the artifacts described in the summary. Alternative percentile calculations are robustness checks, not invalid-solution controls."
+        }
+      ],
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__Fn2e97t/agent/trajectory.json",
+      "trajectory_sha256": "512254e50ecb5da887f73e4518e740fada34c0a5e6af231ff759f41639ee0f8e",
+      "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__Fn2e97t/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "a91dda6b73d95f3b49a84ed5071f688068915336e041b9ea5353c908833a2386",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "6 passed"
+    },
+    {
+      "job": "vq-ver-llm-inference-batching-scheduler-r2-20261009",
+      "arm": "verification",
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "repeat": 2,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": null,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": true,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            37,
+            47,
+            49,
+            51,
+            59,
+            61,
+            65
+          ],
+          "note": "Final plans pass the public constraints including global union (five shapes). Copies with collapsed batches, smaller sequence alignment, a missing ID and a duplicate ID are rejected by the same checker. After later solver changes, saved files are revalidated and a second build is byte-identical; input hashes remain unchanged."
+        }
+      ],
+      "sensitivity_scope": "four_invalid_plan_mutations",
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__8w2qiPm/agent/trajectory.json",
+      "trajectory_sha256": "332ff9d8222d0407949b8ee726350a0da77b5ca26b1772f33d22f98fb44e8a28",
+      "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-llm-inference-batching-scheduler-r2-20261009/llm-inference-batching-scheduler__8w2qiPm/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "4e336375abe88f48db14eb97cd0acad48228d9b7b93b4389bc1096b834028992",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "6 passed"
+    },
+    {
+      "job": "vq-con-build-cython-ext-r1-20261009",
+      "arm": "control",
+      "task": "terminal-bench/build-cython-ext",
+      "repeat": 1,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            61,
+            81,
+            83,
+            89,
+            93,
+            145,
+            147,
+            149,
+            153,
+            157
+          ],
+          "note": "All three required compiled extensions are loaded from system site-packages outside the checkout; NumPy remains 2.3.0. The README example runs, and the public suite passes 18 tests with only the two task-authorized file exclusions. No edits to public test assertions are observed. Source fixes address compatibility errors. Extension comparisons use Python fallbacks and explicit reference calculations. After the final rebuild, tests, the README example and extension checks run again. Broader optional smoke checks have errors in paths requiring absent optional dependencies; the final discloses those limitations."
+        }
+      ],
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-build-cython-ext-r1-20261009/build-cython-ext__ZTtA8Ss/agent/trajectory.json",
+      "trajectory_sha256": "48153789e2b170020d4f870635ee6611478ccf63bc5246fd5b8948ad8179702c",
+      "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-build-cython-ext-r1-20261009/build-cython-ext__ZTtA8Ss/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "8f8bd813b406762911db76a3b2ae041d5305833b64a2eaeef457a882c70dfdc4",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "11 passed"
+    },
+    {
+      "job": "vq-ver-build-cython-ext-r1-20261009",
+      "arm": "verification",
+      "task": "terminal-bench/build-cython-ext",
+      "repeat": 1,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            73,
+            97,
+            103,
+            107,
+            109,
+            121,
+            135,
+            151,
+            157,
+            159,
+            161,
+            169,
+            175,
+            177
+          ],
+          "note": "All three required compiled extensions are loaded from system site-packages outside the checkout; NumPy remains 2.3.0. The README example runs, and the public suite passes 18 tests with only the two task-authorized file exclusions. No edits to public test assertions are observed. Checks compare compiled and Python paths, transcribed/analytic references, and wheel contents. API misuse in the self-test is corrected; final installed checks and tests are rerun. Step 135 retrieves two current upstream implementation files from the explicitly task-supplied project to inspect parser regexes. This is ordinary upstream code reference, not a benchmark solution or hidden test. A late extra in-tree probe uses an unimported submodule and fails; the required snippet and installed extension suite have separate passing evidence."
+        }
+      ],
+      "reference_detail": "Task-authorized repository clone plus upstream gausscode.py and dtnotation.py source; no benchmark-answer retrieval observed.",
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-build-cython-ext-r1-20261009/build-cython-ext__VdAfcu9/agent/trajectory.json",
+      "trajectory_sha256": "22daaeecd50c4cf8f95a545c7bdbe6a295fbf9ae1b6da1f86a88e4fbb2df097e",
+      "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-build-cython-ext-r1-20261009/build-cython-ext__VdAfcu9/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "b4ac51b738fcd16b296c01f4c46038a47f59517572b765ffe28878e4eb725f19",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "11 passed"
+    },
+    {
+      "job": "vq-ope-build-cython-ext-r1-20261009",
+      "arm": "operations",
+      "task": "terminal-bench/build-cython-ext",
+      "repeat": 1,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            67,
+            93,
+            101,
+            107,
+            115,
+            119,
+            121,
+            123,
+            129,
+            131,
+            135
+          ],
+          "note": "All three required compiled extensions are loaded from system site-packages outside the checkout; NumPy remains 2.3.0. The README example runs, and the public suite passes 18 tests with only the two task-authorized file exclusions. No edits to public test assertions are observed. Numerical checks cover crossing detection and basepoint/no-basepoint references. The extended harness then fails because of incorrect API usage; a later broad smoke suite also fails. Its recorded status is already 1 at step 129, although the final calls it still running. The failure is disclosed, but the process-state statement is inaccurate. The installed required tests are rerun after the final source rebuild."
+        }
+      ],
+      "unresolved_failed_check_observed": true,
+      "process_state_claim_inaccurate": true,
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-build-cython-ext-r1-20261009/build-cython-ext__p7UCQpf/agent/trajectory.json",
+      "trajectory_sha256": "e74472ceaf6d4083072edb5ac5f901ccf7f0d0ceceda5e123469cde0099084bc",
+      "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-build-cython-ext-r1-20261009/build-cython-ext__p7UCQpf/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "384ef8efdbc0a05d3abfd7bb76da080ff6df1dfd2be24eeab2be7d99fb8a9648",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "11 passed"
+    },
+    {
+      "job": "vq-ope-build-cython-ext-r2-20261009",
+      "arm": "operations",
+      "task": "terminal-bench/build-cython-ext",
       "repeat": 2,
-      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__TUMs5AX/agent/trajectory.json",
-      "trajectory_sha256": "db9d1cb521b78bcdd040b07393c605f8bbbcdd0bf3f299d7c52eaf34a2563a2b",
       "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
       "eligible_as_independent_capability_evidence": true,
       "task_specific_search_attempt": false,
       "real_entry_check": "partial_pass",
+      "critical_check_coverage": "partial",
       "final_limitations_disclosed": false,
-      "unsupported_completion_claim_observed": false,
-      "unsupported_intermediate_claim_observed": true,
+      "unsupported_completion_claim_observed": true,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            125,
+            127,
+            129,
+            135,
+            137,
+            139,
+            141,
+            143,
+            145,
+            147
+          ],
+          "note": "All three required compiled extensions are loaded from system site-packages outside the checkout; NumPy remains 2.3.0. The README example runs, and the public suite passes 18 tests with only the two task-authorized file exclusions. No edits to public test assertions are observed. The final numerical comparison prints False for no-basepoint writhe and True for basepoint. The reference uses the basepoint formula for both entry points; no subsequent corrected no-basepoint oracle is executed. The final summarizes exact agreement without disclosing this mismatch. Official success does not resolve that self-verification gap. A compileall command also lacks its success marker, while later commands make the shell return normally."
+        }
+      ],
       "unresolved_failed_check_observed": true,
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-build-cython-ext-r2-20261009/build-cython-ext__mLb22QF/agent/trajectory.json",
+      "trajectory_sha256": "e41bc6690edeb4428ed3b0876a8d266729e29844dfc815e26e93dfd0d462c126",
+      "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-build-cython-ext-r2-20261009/build-cython-ext__mLb22QF/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "25c26731e2190107b603c5f8503b181be40f41603c40f6e9f29588ff67ffdcfb",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "11 passed"
+    },
+    {
+      "job": "vq-ver-build-cython-ext-r2-20261009",
+      "arm": "verification",
+      "task": "terminal-bench/build-cython-ext",
+      "repeat": 2,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": true,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            69,
+            71,
+            105,
+            109,
+            111,
+            113,
+            125,
+            127,
+            129,
+            163,
+            167,
+            169,
+            171
+          ],
+          "note": "All three required compiled extensions are loaded from system site-packages outside the checkout; NumPy remains 2.3.0. The README example runs, and the public suite passes 18 tests with only the two task-authorized file exclusions. No edits to public test assertions are observed. The agent repairs source compatibility and verifies compiled functions against Python references. Self-test corrections for zero-based indices and wrapper normalization follow the public implementation. A broad known-value set is expanded to include zero for figure-eight; Python/NumPy comparisons remain but this is weaker than a case-specific independently grounded constant. A perturbed contribution matrix is rejected by the numerical comparison (printed boolean, not a full bad-extension run). Required tests, example and extension checks pass at the end. Cost/usage accounting is partial."
+        }
+      ],
+      "sensitivity_scope": "perturbed_numeric_reference_input",
+      "expectation_narrowing_observed": true,
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-build-cython-ext-r2-20261009/build-cython-ext__8WAwfcA/agent/trajectory.json",
+      "trajectory_sha256": "0e6e7b7a22cf53e11d88186a8a70bbb7c1dcbedc7fb28b1103dcc4df59e32187",
+      "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-build-cython-ext-r2-20261009/build-cython-ext__8WAwfcA/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "8c336587a3bb11d2b8faf5410c6ca91f6bf236653272588ba7af33a54b7a43ee",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "11 passed"
+    },
+    {
+      "job": "vq-con-build-cython-ext-r2-20261009",
+      "arm": "control",
+      "task": "terminal-bench/build-cython-ext",
+      "repeat": 2,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
       "sensitivity_check_observed": false,
       "stale_check_observed": false,
-      "critical_check_coverage": "partial",
       "evidence": [
         {
           "steps": [
+            79,
+            81,
+            83,
+            95,
+            101,
+            125,
+            141,
+            143,
+            145,
+            147,
+            157
+          ],
+          "note": "All three required compiled extensions are loaded from system site-packages outside the checkout; NumPy remains 2.3.0. The README example runs, and the public suite passes 18 tests with only the two task-authorized file exclusions. No edits to public test assertions are observed. An invented figure-eight known-value expectation (1) fails while compiled and Python values both equal 0. The agent removes that known-value assertion (sets it to None) and retains the independent Python comparison and trefoil known value. This narrows the oracle and is not evidence of an independently established figure-eight constant. Final installed tests pass after rebuilding. Cost/usage accounting is partial despite a completed final response."
+        }
+      ],
+      "expectation_narrowing_observed": true,
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-build-cython-ext-r2-20261009/build-cython-ext__qL2BBUt/agent/trajectory.json",
+      "trajectory_sha256": "3a2caa54b6998d9d99861acf2c6b1a150921caa29ccd3ce3346e71c2d9b7ceb2",
+      "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-build-cython-ext-r2-20261009/build-cython-ext__qL2BBUt/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "288e48b524821d4ac33e8d8b8dbe27ca5ada894dd7fe9bcf9965522bcdb65190",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "11 passed"
+    },
+    {
+      "job": "vq-ver-cancel-async-tasks-r1-20261009",
+      "arm": "verification",
+      "task": "terminal-bench/cancel-async-tasks",
+      "repeat": 1,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": true,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            25,
             29,
-            31,
-            33,
-            35,
-            37,
-            39,
-            41
+            43,
+            45,
+            49,
+            55,
+            65,
+            69,
+            71,
+            75
           ],
-          "note": "A six-layer real Llama harness compares per-microbatch forward/backward activations and selected parameter gradients against a full-batch reference. It exposes and prompts fixes for broadcast_object_list result handling and mutually exclusive input_ids/inputs_embeds. Backward hook captures are changed from overwrites to lists; the implementation then changes backward processing to forward microbatch order, eliminating observed per-item mismatches. This is real defect-finding and repair evidence."
-        },
+          "note": "Actual cancellation tests expose interrupted cleanup under repeated cancellation. After changing the drain logic, functional, leak, cancellation and subprocess SIGINT checks pass. The same suites reject naive gather and cancel-without-wait implementations. Tests are rerun after the final source annotation change. Initial test-factory and mixed-type sorting mistakes are corrected rather than deleting cleanup assertions."
+        }
+      ],
+      "sensitivity_scope": "two_invalid_runner_implementations",
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-cancel-async-tasks-r1-20261009/cancel-async-tasks__amy3eoc/agent/trajectory.json",
+      "trajectory_sha256": "9708c625754f6a232f39086f77a9282f207bb17edd68e0acdf50f6bf46655acf",
+      "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-cancel-async-tasks-r1-20261009/cancel-async-tasks__amy3eoc/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "9e456ebc4478cbda383fc7871f51a75f64fce94cb44b1cbe1c39645acd96ef8e",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "6 passed"
+    },
+    {
+      "job": "vq-ope-cancel-async-tasks-r1-20261009",
+      "arm": "operations",
+      "task": "terminal-bench/cancel-async-tasks",
+      "repeat": 1,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "partial_pass",
+      "critical_check_coverage": "partial",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
         {
           "steps": [
+            9,
+            25,
+            27,
+            31,
+            35,
             39,
             41,
             43,
             45
           ],
-          "note": "The intermediate statement that parameter gradients match conflicts with observed embedding/head gradient failures. Two-rank tied-weight parameter checks remain failing after the ordering fix. The next test changes tie_word_embeddings from True to False, and both ranks report success; the tied case is not repaired or rechecked. The harness skips unobserved layers, does not independently assert the layer partition, and labels its aggregate numerical flag layers_per_rank_ok. It also does not check the final norm gradient despite a broader comment. The final model call exhausts the wall-clock budget and produces no final summary or disclosure. Its cost and token accounting are incomplete."
+          "note": "Final-file checks pass bounded execution, direct cancellation, errors and direct asyncio.run subprocess SIGINT with awaited cleanup. An outer-gather SIGINT test still interrupts cleanup. The final discloses that failure, so the evidence supports only selected entry points. Its further assertion that this limitation affects any coroutine runner is not established."
         }
       ],
-      "official_failure_notes": "The first verifier attempt completes actual pytest execution with reward 1: all four tests pass. This official pass does not repair the unreported tied-weight self-test failure or make incomplete accounting complete.",
+      "unresolved_failed_check_observed": true,
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-cancel-async-tasks-r1-20261009/cancel-async-tasks__ikKRqnw/agent/trajectory.json",
+      "trajectory_sha256": "4c022734a5435fd0e22aeb96700e5c954029b911a32756f88782cb884486bda4",
+      "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-cancel-async-tasks-r1-20261009/cancel-async-tasks__ikKRqnw/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "a5491fef69c9a8f76b0ef43b620978abe560df28f3db527bd5ac451053e28c25",
       "score_valid_for_capability_comparison": true,
       "score_classification": "scored_pass",
-      "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-torch-pipeline-parallelism-r2-20261009/torch-pipeline-parallelism__TUMs5AX/verifier/test-stdout.txt",
-      "verifier_stdout_sha256": "931c5d89033bae35cf65965dd76519f77eb2b899ac50ebbfc96564fd755e4e73"
+      "official_test_summary": "6 passed"
+    },
+    {
+      "job": "vq-con-cancel-async-tasks-r1-20261009",
+      "arm": "control",
+      "task": "terminal-bench/cancel-async-tasks",
+      "repeat": 1,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            9,
+            11,
+            13,
+            15,
+            17,
+            23,
+            25,
+            27
+          ],
+          "note": "Tests cover bounded execution, direct cancellation, real subprocess SIGINT under asyncio.run and a raw loop, awaited cleanup and handler restoration. A KeyboardInterrupt test raises the concurrency limit so all jobs whose cleanup is expected actually start; other limit checks remain. Tests are rerun after the final annotation edit. No deliberate invalid implementation is tested."
+        }
+      ],
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-cancel-async-tasks-r1-20261009/cancel-async-tasks__ND8xrVi/agent/trajectory.json",
+      "trajectory_sha256": "59f208eb5474be2af7bb7733091bc3b0c8622ea25c55b37999d059e9037e129f",
+      "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-cancel-async-tasks-r1-20261009/cancel-async-tasks__ND8xrVi/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "1047403c7b1e524413e5011d8179eb2b1a436e5fce02c069c39ebaea801ecb76",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "6 passed"
+    },
+    {
+      "job": "vq-con-cancel-async-tasks-r2-20261009",
+      "arm": "control",
+      "task": "terminal-bench/cancel-async-tasks",
+      "repeat": 2,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            9,
+            11,
+            17,
+            19,
+            21,
+            23,
+            25,
+            31,
+            33,
+            35,
+            39,
+            43
+          ],
+          "note": "A real SIGINT check initially shows no cleanup; diagnosis finds repeated cancellation interrupting awaited cleanup. The implementation is rewritten, and direct cancellation plus subprocess signals show matching started/cleaned IDs. Tests are rerun after later source edits. The final explicitly limits very early and repeated SIGINT behavior. A type-hint comparison prints False because None differs from NoneType; it is not a failed runtime cleanup test."
+        }
+      ],
+      "trajectory_path": "jobs/verification-quality-control-source/jobs/vq-con-cancel-async-tasks-r2-20261009/cancel-async-tasks__fGR6umA/agent/trajectory.json",
+      "trajectory_sha256": "b33170cbd33bd77e240cf98f26e1269c84f6569e20812b468070c57d2d8ff60b",
+      "verifier_stdout_path": "jobs/verification-quality-control-source/jobs/vq-con-cancel-async-tasks-r2-20261009/cancel-async-tasks__fGR6umA/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "1047403c7b1e524413e5011d8179eb2b1a436e5fce02c069c39ebaea801ecb76",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "6 passed"
+    },
+    {
+      "job": "vq-ope-cancel-async-tasks-r2-20261009",
+      "arm": "operations",
+      "task": "terminal-bench/cancel-async-tasks",
+      "repeat": 2,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": false,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            7,
+            9,
+            11,
+            15,
+            17,
+            19,
+            23
+          ],
+          "note": "Functional bounded execution, direct cancellation and wait_for cleanup pass. A shell-background SIGINT driver times out at the 120-second tool limit. Replacing it with Python subprocess signaling produces an interrupted process with all three started IDs cleaned. Final basic checks and signature inspection pass. The final limits repeated SIGINT behavior."
+        }
+      ],
+      "trajectory_path": "jobs/verification-quality-operations-source/jobs/vq-ope-cancel-async-tasks-r2-20261009/cancel-async-tasks__C9zjrGs/agent/trajectory.json",
+      "trajectory_sha256": "2d51c8161d70347fbbee49e5ead2215cc3a97be25cc6f78053f945ba8b323ba6",
+      "verifier_stdout_path": "jobs/verification-quality-operations-source/jobs/vq-ope-cancel-async-tasks-r2-20261009/cancel-async-tasks__C9zjrGs/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "9e456ebc4478cbda383fc7871f51a75f64fce94cb44b1cbe1c39645acd96ef8e",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "6 passed"
+    },
+    {
+      "job": "vq-ver-cancel-async-tasks-r2-20261009",
+      "arm": "verification",
+      "task": "terminal-bench/cancel-async-tasks",
+      "repeat": 2,
+      "reference_audit": "reviewed_no_confirmed_external_task_answer_exposure",
+      "eligible_as_independent_capability_evidence": true,
+      "task_specific_search_attempt": false,
+      "real_entry_check": "pass_selected_cases",
+      "critical_check_coverage": "covered_public_core_in_selected_cases",
+      "final_limitations_disclosed": true,
+      "unsupported_completion_claim_observed": false,
+      "sensitivity_check_observed": true,
+      "stale_check_observed": false,
+      "evidence": [
+        {
+          "steps": [
+            9,
+            13,
+            17,
+            19,
+            23,
+            25,
+            27,
+            31,
+            35,
+            37,
+            39
+          ],
+          "note": "Real SIGINT initially loses cleanup and passes after source repairs. Functional concurrency, raw-loop and asyncio.run subprocess signals, and 200-job stress checks pass on the final file. The same SIGINT oracle rejects a naive gather/double-cancel implementation. A test forbidding a queued job from starting after a sibling error is changed to require cleanup of every started job; that scheduling prohibition is not in the public contract. The final shell summary uses pipelines without pipefail, but preceding individual runs and visible pass markers support the reported suites."
+        }
+      ],
+      "sensitivity_scope": "invalid_double_cancel_runner",
+      "trajectory_path": "jobs/verification-quality-verification-source/jobs/vq-ver-cancel-async-tasks-r2-20261009/cancel-async-tasks__MNp38Lb/agent/trajectory.json",
+      "trajectory_sha256": "b02fcc79dc8824ae09cdd39045ebc3f482106516624072b191c671f61112e4fc",
+      "verifier_stdout_path": "jobs/verification-quality-verification-source/jobs/vq-ver-cancel-async-tasks-r2-20261009/cancel-async-tasks__MNp38Lb/verifier/test-stdout.txt",
+      "verifier_stdout_sha256": "1f0c6ba8656676ed4bab8c747c70a2e98c4629615edf37879264a7a416dc8252",
+      "score_valid_for_capability_comparison": true,
+      "score_classification": "scored_pass",
+      "official_test_summary": "6 passed"
     }
-  ]
+  ],
+  "field_notes": {
+    "pass_selected_cases": "Observed real entry execution and checks; not proof of exhaustive correctness.",
+    "sensitivity_check_observed": "Includes graded scopes: invalid implementations/artifacts, baseline metric thresholds, or a perturbed numeric input. Compare scope, not just boolean counts.",
+    "final_limitations_disclosed": "Null means no unresolved core limitation was identified that required a disclosure judgment.",
+    "unsupported_completion_claim_observed": "A material final verification/completion claim exceeds or contradicts observed evidence; narrower caveats are retained separately.",
+    "reference_audit": "No confirmed answer exposure is a trace-based observation, not proof of no possible exposure."
+  }
 }
diff --git a/benchmarks/harbor/results/verification-quality-integrity-20261009.json b/benchmarks/harbor/results/verification-quality-integrity-20261009.json
new file mode 100644
index 0000000..9cadcef
--- /dev/null
+++ b/benchmarks/harbor/results/verification-quality-integrity-20261009.json
@@ -0,0 +1,72 @@
+{
+  "schema_version": 1,
+  "checked_at": "2026-10-11T14:05:59.246153+00:00",
+  "plan_sha256": "b3968a10a29507b72c62f6f886a6af4f982373e15ca6103da21dd288362396c1",
+  "registered_plan_configs_prompts_task_file_hashes": "passed_by_collect_comparison",
+  "all_trial_manifest_source_hashes_and_installed_wheel_hashes": "passed_by_collect_comparison",
+  "only_runtime_prompt_differs": true,
+  "equal_agent_ast_without_headless_prompt_sha256": "147c1d89b6f667415859714f3c5f52c6b4ba7ac856c5b002a510af158e5283cd",
+  "other_runtime_and_adapter_python_files_equal": true,
+  "version_metadata_note": "Generated _version.py is excluded from source equivalence; each installed wheel is matched to its recorded arm.",
+  "atif_trajectories_validated": 24,
+  "official_test_summaries_checked": 18,
+  "native_exceptions": 0,
+  "verifier_retry_policy": "All 24 retain original per-attempt limit; only prior timeout attempts trigger a retry; at most 3 attempts.",
+  "verifier_dependency_profiles": "not_configured for all 24",
+  "preflight_jobs_before_model_work": [
+    {
+      "job": "vq-install-control-20261009",
+      "finished_at": "2026-10-09T14:48:59.034610+00:00"
+    },
+    {
+      "job": "vq-install-verification-20261009",
+      "finished_at": "2026-10-09T14:50:18.301552+00:00"
+    },
+    {
+      "job": "vq-install-operations-20261009",
+      "finished_at": "2026-10-09T14:51:38.884784+00:00"
+    }
+  ],
+  "images": [
+    {
+      "task": "terminal-bench/torch-pipeline-parallelism",
+      "image": "alexgshaw/torch-pipeline-parallelism:20251031",
+      "image_id": "sha256:3cb7b39d86f1704b20e668851da0b08b34b3270c01d583f136d65190984a8d1b",
+      "repo_digests": [
+        "alexgshaw/torch-pipeline-parallelism@sha256:3cb7b39d86f1704b20e668851da0b08b34b3270c01d583f136d65190984a8d1b"
+      ],
+      "size": 29725531,
+      "post_run_image_id": "sha256:3cb7b39d86f1704b20e668851da0b08b34b3270c01d583f136d65190984a8d1b"
+    },
+    {
+      "task": "terminal-bench/llm-inference-batching-scheduler",
+      "image": "alexgshaw/llm-inference-batching-scheduler:20251031",
+      "image_id": "sha256:19e79aa49be4e55dccca1dde966515ebe6852314b6010bcb9230afb48b996774",
+      "repo_digests": [
+        "alexgshaw/llm-inference-batching-scheduler@sha256:19e79aa49be4e55dccca1dde966515ebe6852314b6010bcb9230afb48b996774"
+      ],
+      "size": 44188631,
+      "post_run_image_id": "sha256:19e79aa49be4e55dccca1dde966515ebe6852314b6010bcb9230afb48b996774"
+    },
+    {
+      "task": "terminal-bench/build-cython-ext",
+      "image": "alexgshaw/build-cython-ext:20251031",
+      "image_id": "sha256:3612a38fadb89a96f74a1a951fb0b0af734198fd160571eeaba6401593234594",
+      "repo_digests": [
+        "alexgshaw/build-cython-ext@sha256:3612a38fadb89a96f74a1a951fb0b0af734198fd160571eeaba6401593234594"
+      ],
+      "size": 303222492,
+      "post_run_image_id": "sha256:3612a38fadb89a96f74a1a951fb0b0af734198fd160571eeaba6401593234594"
+    },
+    {
+      "task": "terminal-bench/cancel-async-tasks",
+      "image": "alexgshaw/cancel-async-tasks:20251031",
+      "image_id": "sha256:84c7fae6b256dcc56a350790e2a9715eefc7dad662a9d8e8a472363aa71ef18d",
+      "repo_digests": [
+        "alexgshaw/cancel-async-tasks@sha256:84c7fae6b256dcc56a350790e2a9715eefc7dad662a9d8e8a472363aa71ef18d"
+      ],
+      "size": 44171676,
+      "post_run_image_id": "sha256:84c7fae6b256dcc56a350790e2a9715eefc7dad662a9d8e8a472363aa71ef18d"
+    }
+  ]
+}
diff --git a/benchmarks/harbor/scripts/collect_comparison.py b/benchmarks/harbor/scripts/collect_comparison.py
new file mode 100644
index 0000000..a9d501b
--- /dev/null
+++ b/benchmarks/harbor/scripts/collect_comparison.py
@@ -0,0 +1,201 @@
+"""Collect a completed, audited comparison without running models or verifiers.
+
+Run from the repository root. Local job artifacts and frozen source worktrees
+must still exist. Costs are estimates; missing accounting remains incomplete.
+"""
+
+import argparse
+from collections import Counter
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+
+from run_comparison import digest, validate_plan
+
+
+def accounting(metrics):
+    extra = metrics.get("extra", {})
+    total = metrics.get("total_cost_usd")
+    return {
+        "known_cost_usd": total if total is not None else extra.get("known_cost_usd"),
+        "cost_complete": total is not None and not extra.get("cost_is_partial", False),
+        "usage_complete": extra.get("usage_complete", True) and all(
+            metrics.get(key) is not None for key in (
+                "total_prompt_tokens", "total_completion_tokens", "total_cached_tokens",
+            )
+        ),
+    }
+
+
+def classify(result, audit):
+    rewards = (result.get("verifier_result") or {}).get("rewards") or {}
+    reward = rewards.get("reward")
+    exception = (result.get("exception_info") or {}).get("exception_type")
+    native = "execution_error" if exception else "scored" if reward is not None else "unscored"
+    status = native
+    if audit.get("score_classification") == "verifier_dependency_error":
+        status = "verifier_error"
+        reward = None
+    return {
+        "native_status": native, "native_reward": rewards.get("reward"),
+        "raw_verifier_rewards": rewards, "exception_type": exception,
+        "status": status, "reward": reward,
+        "eligible_as_independent_capability_evidence": status == "scored"
+        and audit.get("score_valid_for_capability_comparison") is True
+        and audit.get("eligible_as_independent_capability_evidence") is True,
+    }
+
+
+def summarize(rows):
+    costs = [r["known_cost_usd"] for r in rows if r["known_cost_usd"] is not None]
+    return {
+        "trials": len(rows), "statuses": dict(Counter(r["status"] for r in rows)),
+        "passes": sum(r["status"] == "scored" and r["reward"] == 1 for r in rows),
+        "scored_failures": sum(r["status"] == "scored" and r["reward"] == 0 for r in rows),
+        "known_cost_usd": round(sum(costs), 9) if costs else None,
+        "cost_complete": bool(rows) and all(r["cost_complete"] for r in rows),
+        "usage_complete": bool(rows) and all(r["usage_complete"] for r in rows),
+        "incomplete_accounting_trials": sum(not r["cost_complete"] for r in rows),
+        "agent_execution_seconds": sum(r["agent_execution_seconds"] for r in rows),
+        "model_attempts": sum(r["model_attempts"] for r in rows),
+        "incomplete_model_attempts": sum(r["incomplete_model_attempts"] for r in rows),
+        "tool_timeouts": sum(r["tool_timeouts"] for r in rows),
+    }
+
+
+def common_cases(rows, arms):
+    eligible = {
+        arm: {(r["task"], r["repeat"]) for r in rows if r["arm"] == arm
+              and r["eligible_as_independent_capability_evidence"]}
+        for arm in arms
+    }
+    return set.intersection(*eligible.values())
+
+
+def seconds(interval):
+    return (datetime.fromisoformat(interval["finished_at"]) -
+            datetime.fromisoformat(interval["started_at"])).total_seconds()
+
+
+def load_checked(path, expected_hash):
+    if digest(path) != expected_hash:
+        raise ValueError(f"Evidence hash mismatch: {path}")
+    return json.loads(path.read_text())
+
+
+def collect(root, plan_path, audit_path):
+    plan = json.loads(plan_path.read_text())
+    validate_plan(root, plan_path, plan)
+    registration = json.loads((root / plan["log_dir"] / "registration.json").read_text())
+    if registration != {"plan_sha256": digest(plan_path), "plan": plan}:
+        raise ValueError("Registered plan changed")
+    audit = json.loads(audit_path.read_text())
+    audits = {a["job"]: a for a in audit["trials"]}
+    expected_jobs = {j["job_name"] for j in plan["queue"]}
+    if (audit["status"] != "completed" or set(audits) != expected_jobs
+            or len(audit["trials"]) != len(expected_jobs)):
+        raise ValueError("Every planned trial needs exactly one completed audit")
+    tasks = {t["name"]: t for t in plan["tasks"]}
+    rows = []
+    for job in plan["queue"]:
+        arm = plan["arms"][job["arm"]]
+        source = root / arm["source_root"]
+        state = json.loads((root / plan["log_dir"] / (job["job_name"] + ".json")).read_text())
+        if state["status"] != "finished":
+            raise ValueError(f"Unfinished job: {job['job_name']}")
+        results = list((source / "jobs" / job["job_name"]).glob("*/result.json"))
+        if len(results) != 1:
+            raise ValueError(f"Expected one trial: {job['job_name']}")
+        result_path = results[0]
+        result = json.loads(result_path.read_text())
+        if not result.get("finished_at"):
+            raise ValueError(f"Unfinished trial: {result_path}")
+        evidence = audits[job["job_name"]]
+        trajectory_path = result_path.parent / "agent/trajectory.json"
+        if root / evidence["trajectory_path"] != trajectory_path:
+            raise ValueError("Audit points to a different trial")
+        trajectory = load_checked(trajectory_path, evidence["trajectory_sha256"])
+        stdout = result_path.parent / "verifier/test-stdout.txt"
+        if (root / evidence["verifier_stdout_path"] != stdout
+                or digest(stdout) != evidence["verifier_stdout_sha256"]):
+            raise ValueError(f"Verifier evidence changed: {stdout}")
+        bootstrap = json.loads((result_path.parent / "agent/bootstrap.json").read_text())
+        manifest_path = source / "jobs" / (job["job_name"] + "-input/manifest.json")
+        manifest = json.loads(manifest_path.read_text())
+        if manifest["commit"] != arm["source_ref"]:
+            raise ValueError("Trial source revision differs from the plan")
+        wheel = Path(result["config"]["agent"]["kwargs"]["wheel_path"])
+        if not (digest(wheel) == manifest["wheel_sha256"] == bootstrap["wheel_sha256"]):
+            raise ValueError("Installed wheel differs from the recorded build")
+        for name, expected in manifest["source_sha256"].items():
+            prefix = "benchmarks/harbor/src" if name.startswith("harbor_adapter/") else "src"
+            if digest(source / prefix / name) != expected:
+                raise ValueError(f"Trial runtime source changed: {name}")
+        task = tasks[job["task"]]
+        if result["task_id"]["ref"] != task["ref"]:
+            raise ValueError("Trial task revision differs from the plan")
+        retry_path = result_path.parent / "verifier/retry-summary.json"
+        retry = json.loads(retry_path.read_text())
+        attempts = retry["attempts"]
+        if not (retry["timeout_sec"] == task["verifier_timeout_seconds"]
+                and retry["max_attempts"] == 3 and 1 <= len(attempts) <= 3
+                and all(a["status"] == "timeout" for a in attempts[:-1])):
+            raise ValueError("Verifier retries violate the registered policy")
+        steps = trajectory["steps"]
+        model_steps = [s for s in steps if s.get("llm_call_count")]
+        observations = [o for s in steps for o in s.get("observation", {}).get("results", [])]
+        metrics = trajectory["final_metrics"]
+        rows.append({
+            **job, "cohort": task["cohort"], **classify(result, evidence),
+            **accounting(metrics), "metrics": metrics,
+            "launcher_exit_code": state["exit_code"],
+            "result_path": str(result_path.relative_to(root)),
+            "result_sha256": digest(result_path), "task_checksum": result["task_checksum"],
+            "trajectory_path": str(trajectory_path.relative_to(root)),
+            "trajectory_sha256": digest(trajectory_path),
+            "verifier_stdout_sha256": digest(stdout),
+            "source_ref": manifest["commit"], "wheel_sha256": manifest["wheel_sha256"],
+            "terminal": trajectory.get("extra", {}).get("terminal"),
+            "started_at": result["started_at"], "finished_at": result["finished_at"],
+            "agent_setup_seconds": seconds(result["agent_setup"]),
+            "agent_execution_seconds": seconds(result["agent_execution"]),
+            "verifier_seconds": seconds(result["verifier"]),
+            "verifier_attempts": retry, "verifier_preflight": bootstrap["verifier_preflight"],
+            "model_attempts": sum(s["llm_call_count"] for s in model_steps),
+            "incomplete_model_attempts": sum(bool(s.get("extra", {}).get("incomplete")) for s in steps),
+            "truncated_responses": sum(s.get("extra", {}).get("stop_reason") == "max_tokens" for s in steps),
+            "tool_timeouts": sum("[command timed out after" in str(o.get("content", "")) for o in observations),
+            "audit": evidence,
+        })
+    arms = list(plan["arms"])
+    common = common_cases(rows, arms)
+    paired = [r for r in rows if (r["task"], r["repeat"]) in common]
+    held_out = [r for r in paired if r["cohort"] == "held_out_from_prior_tuning"]
+    return {
+        "schema_version": 2, "status": "completed",
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "experiment_plan": str(plan_path.relative_to(root)),
+        "experiment_plan_sha256": digest(plan_path),
+        "audit_path": str(audit_path.relative_to(root)), "audit_sha256": digest(audit_path),
+        "cost_note": "Token-based estimates, not invoices. Incomplete totals are known lower bounds, never zero-imputed.",
+        "summary": {arm: summarize([r for r in rows if r["arm"] == arm]) for arm in arms},
+        "common_eligible_cases": [dict(task=t, repeat=r) for t, r in sorted(common)],
+        "common_eligible_summary": {arm: summarize([r for r in paired if r["arm"] == arm]) for arm in arms},
+        "held_out_common_summary": {arm: summarize([r for r in held_out if r["arm"] == arm]) for arm in arms},
+        "trials": rows,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--plan", type=Path, required=True)
+    parser.add_argument("--audit", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    result = collect(Path.cwd(), args.plan.resolve(), args.audit.resolve())
+    args.output.write_text(json.dumps(result, indent=2) + "\n")
+    print(json.dumps(result["summary"], indent=2))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/benchmarks/harbor/tests/test_comparison_results.py b/benchmarks/harbor/tests/test_comparison_results.py
new file mode 100644
index 0000000..ed05c7d
--- /dev/null
+++ b/benchmarks/harbor/tests/test_comparison_results.py
@@ -0,0 +1,67 @@
+"""Prevent infrastructure failures and incomplete accounting from becoming wins."""
+
+import importlib.util
+from pathlib import Path
+import sys
+
+import pytest
+
+
+scripts = Path(__file__).parents[1] / "scripts"
+spec = importlib.util.spec_from_file_location("collect_comparison", scripts / "collect_comparison.py")
+collector = importlib.util.module_from_spec(spec)
+sys.path.insert(0, str(scripts))
+try:
+    spec.loader.exec_module(collector)
+finally:
+    sys.path.remove(str(scripts))
+
+
+def test_native_zero_preserved_when_tests_never_started():
+    result = {"verifier_result": {"rewards": {"reward": 0.0}}}
+    audit = {"score_classification": "verifier_dependency_error",
+             "score_valid_for_capability_comparison": False}
+    row = collector.classify(result, audit)
+    assert row["native_status"] == "scored"
+    assert row["native_reward"] == 0
+    assert row["status"] == "verifier_error"
+    assert row["reward"] is None
+    assert not row["eligible_as_independent_capability_evidence"]
+
+
+def test_missing_usage_is_not_zero_cost_or_complete():
+    unknown = collector.accounting({})
+    assert unknown == {"known_cost_usd": None, "cost_complete": False, "usage_complete": False}
+    partial = collector.accounting({"extra": {"known_cost_usd": 0.12, "cost_is_partial": True}})
+    assert partial == {"known_cost_usd": 0.12, "cost_complete": False, "usage_complete": False}
+
+
+def test_accounted_zero_is_distinct_from_missing_cost():
+    metrics = {"total_cost_usd": 0.0, "total_prompt_tokens": 0,
+               "total_completion_tokens": 0, "total_cached_tokens": 0}
+    assert collector.accounting(metrics) == {
+        "known_cost_usd": 0.0, "cost_complete": True, "usage_complete": True,
+    }
+
+
+def test_common_comparison_excludes_invalid_controls_and_exposed_candidates():
+    rows = [dict(arm=a, task=t, repeat=1, eligible_as_independent_capability_evidence=True)
+            for a in ("control", "verification", "operations") for t in ("infra", "exposed", "valid")]
+    rows[0]["eligible_as_independent_capability_evidence"] = False
+    rows[4]["eligible_as_independent_capability_evidence"] = False
+    assert collector.common_cases(rows, ["control", "verification", "operations"]) == {("valid", 1)}
+
+
+def test_unreviewed_scores_are_not_eligible():
+    row = collector.classify({"verifier_result": {"rewards": {"reward": 1}}}, {})
+    assert row["reward"] == 1
+    assert not row["eligible_as_independent_capability_evidence"]
+
+
+def test_evidence_modified_since_review_is_rejected(tmp_path):
+    path = tmp_path / "trace.json"
+    path.write_text('{"value": 1}')
+    reviewed_hash = collector.digest(path)
+    path.write_text('{"value": 2}')
+    with pytest.raises(ValueError, match="Evidence hash mismatch"):
+        collector.load_checked(path, reviewed_hash)