From 31d60fe3846b5f4da54f618f1648fd6d5c566bd8 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Sun, 11 Oct 2026 07:38:56 -0700 Subject: [PATCH 1/5] test(benchmarks): register a 12-task main development baseline --- .../configs/dev12-baseline-20261011/README.md | 84 +++ .../adaptive-rejection-sampler.json | 22 + .../build-pov-ray.json | 22 + .../cobol-modernization.json | 22 + .../constraints-scheduling.json | 22 + .../custom-memory-heap-crash.json | 22 + .../db-wal-recovery.json | 22 + .../dev12-baseline-20261011/experiment.json | 504 ++++++++++++++++++ .../fix-code-vulnerability.json | 22 + .../git-multibranch.json | 22 + .../dev12-baseline-20261011/install-only.json | 66 +++ .../dev12-baseline-20261011/main-prompt.txt | 9 + .../multi-source-data-merger.json | 22 + .../nginx-request-logging.json | 22 + .../pytorch-model-cli.json | 22 + .../query-optimize.json | 22 + .../dev12-baseline-20261011/selection.json | 96 ++++ benchmarks/harbor/scripts/run_comparison.py | 155 ++++++ benchmarks/harbor/tests/test_comparison.py | 57 ++ 19 files changed, 1235 insertions(+) create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/README.md create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/adaptive-rejection-sampler.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/build-pov-ray.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/cobol-modernization.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/constraints-scheduling.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/custom-memory-heap-crash.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/db-wal-recovery.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/experiment.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/fix-code-vulnerability.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/git-multibranch.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/install-only.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/main-prompt.txt create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/multi-source-data-merger.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/nginx-request-logging.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/pytorch-model-cli.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/query-optimize.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/selection.json create mode 100644 benchmarks/harbor/scripts/run_comparison.py create mode 100644 benchmarks/harbor/tests/test_comparison.py diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/README.md b/benchmarks/harbor/configs/dev12-baseline-20261011/README.md new file mode 100644 index 0000000..1e3ebd8 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/README.md @@ -0,0 +1,84 @@ +# Twelve-task development baseline from main + +This cohort adds twelve tasks to the 22 previously exercised tasks. Selection +was fixed by public task names and desired coverage before downloading task +packages, reading their instructions, or seeing outcomes. It is a purposive +**development** cohort, not a representative random sample of Terminal-Bench. +The other 55 tasks are left unopened by this experiment. + +The runtime is main at `2eb99b608dfb39464d17d109a4a8f11afda607a9`, isolated in +`jobs/dev12-baseline-main-source`. This branch records configuration and evidence; +it does not change the agent or adapter. Neither prompt candidate from PR #48 +is included. The fixed dataset revision is +`sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a`. + +[Selection](selection.json) records the existing 22 tasks, selected revisions and +rationales. The [registered plan](experiment.json) records runtime, prompt, +configuration and task-file hashes, work limits, preflight and queue order. + +| Task | Coverage | Work budget (seconds) | +| --- | --- | ---: | +| `build-pov-ray` | Legacy native compilation | 11,820 | +| `cobol-modernization` | Language migration and behavioral compatibility | 720 | +| `custom-memory-heap-crash` | C++ memory debugging | 1,620 | +| `fix-code-vulnerability` | Security repair and behavior preservation | 720 | +| `nginx-request-logging` | Service configuration and observable behavior | 720 | +| `git-multibranch` | Git deployment and service integration | 720 | +| `db-wal-recovery` | Database recovery and data integrity | 720 | +| `multi-source-data-merger` | Data integration and conflict resolution | 720 | +| `query-optimize` | SQL correctness and performance | 720 | +| `adaptive-rejection-sampler` | Numerical algorithms and statistical checks | 720 | +| `constraints-scheduling` | Calendar constraints and temporal reasoning | 1,020 | +| `pytorch-model-cli` | C inference and model CLI contracts | 720 | + +All trials use `deepseek/deepseek-flash` at the official DeepSeek +Anthropic-compatible endpoint, 100 replies and 65,536 output tokens per request. +Temperature and reasoning effort are unset. Each task runs once. Work budgets +follow the existing policy: native task agent timeout minus 180 seconds. +POV-Ray has a 12,000-second native limit; it is retained rather than capped to +match the shorter tasks. Setup independently allows 1,800 seconds. + +The queue is fixed by ascending work budget, preserving selection order on ties. +Consecutive pairs finish before the next pair starts; at most two trials run +concurrently. There are no outcome-based task replacements or whole-trial +retries. Verifiers retain their original task limits and may retry only after a +timeout, at most three total attempts using the same agent output. + +## Run and preserve evidence + +Run from the repository root after creating the exact source checkout. Pass +credentials through the process environment, never configuration or logs. Set +`ANTHROPIC_BASE_URL=https://api.deepseek.com/anthropic` and leave +`ANTHROPIC_MODEL` unset so the recorded configuration selects the model. + +```bash +python benchmarks/harbor/scripts/run_comparison.py \ + --plan benchmarks/harbor/configs/dev12-baseline-20261011/experiment.json \ + --phase preflight + +python benchmarks/harbor/scripts/run_comparison.py \ + --plan benchmarks/harbor/configs/dev12-baseline-20261011/experiment.json \ + --phase model +``` + +The reusable orchestrator calls only the standard +`uv run --project benchmarks/harbor python -m harbor_adapter run --config ...` +workflow from the frozen repository root. It creates an immutable registration, +requires all twelve install-only results before model work, and refuses to retry +a job already recorded as started. Source snapshots and wheel manifests remain +in the normal Harbor job-input directories. + +All twelve selected verifier dependency profiles are explicitly +`not_configured`. Agent installation preflight does **not** certify verifier +dependencies. Preserve original images, package sources and official scripts. +Record setup failures, verifier errors and scored failures separately; preserve +raw rewards even when an unchanged verifier writes zero before tests start. + +Inspect only public metadata/instructions before every model output freezes. +Hashing task files does not require inspecting their contents. Afterwards, audit +traces for external benchmark answers, verification quality and incomplete +accounting. A single observed pass or failure is not a stability estimate. +Keep these twelve results separate from previous pilot20 and prompt comparisons. + +For a later reproduction, use new job names and a new registration directory; +never overwrite this baseline's artifacts. diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/adaptive-rejection-sampler.json b/benchmarks/harbor/configs/dev12-baseline-20261011/adaptive-rejection-sampler.json new file mode 100644 index 0000000..381d3f9 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/adaptive-rejection-sampler.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/build-pov-ray.json b/benchmarks/harbor/configs/dev12-baseline-20261011/build-pov-ray.json new file mode 100644 index 0000000..c228cfd --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/build-pov-ray.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 11820 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/cobol-modernization.json b/benchmarks/harbor/configs/dev12-baseline-20261011/cobol-modernization.json new file mode 100644 index 0000000..459c25e --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/cobol-modernization.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/constraints-scheduling.json b/benchmarks/harbor/configs/dev12-baseline-20261011/constraints-scheduling.json new file mode 100644 index 0000000..4958be1 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/constraints-scheduling.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 1020 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/custom-memory-heap-crash.json b/benchmarks/harbor/configs/dev12-baseline-20261011/custom-memory-heap-crash.json new file mode 100644 index 0000000..3974e78 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/custom-memory-heap-crash.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 1620 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/db-wal-recovery.json b/benchmarks/harbor/configs/dev12-baseline-20261011/db-wal-recovery.json new file mode 100644 index 0000000..3781ed1 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/db-wal-recovery.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/experiment.json b/benchmarks/harbor/configs/dev12-baseline-20261011/experiment.json new file mode 100644 index 0000000..364ef4b --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/experiment.json @@ -0,0 +1,504 @@ +{ + "schema_version": 1, + "registered_at": "2026-10-11T14:37:12.539348+00:00", + "experiment": "dev12-main-baseline-20261011", + "selection_file": "selection.json", + "selection_sha256": "cd51acea6592c01ce9ec8cde34cd46a4965a4164950194f2de7e8ca54649bd29", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "arms": { + "main": { + "source_root": "jobs/dev12-baseline-main-source", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "prompt_file": "main-prompt.txt", + "prompt_sha256": "37c4d9d44640c0051dc5a20e5954a067b98e8c51be07a48f080aa059e725c55d", + "prompt_words": 385 + } + }, + "tasks": [ + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb", + "selection_rationale": "Native compilation and build tooling", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 12000, + "work_budget_seconds": 11820, + "verifier_timeout_seconds": 12000.0, + "category": "software-engineering", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/build-pov-ray:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "38380f64bc631916e264f80ce4d37feedbc783310001475b367d03989d114fdf", + "environment/Dockerfile": "44f2b5fc7311d8f336daaaca99dd4216b49355e67f50b494ebacf1b0f7c14e92", + "environment/deps/illum1.pov": "5a1f7f53ee0730c4923fcd21cd4a9c34719024d8557920441551ba7d96e75dc4", + "instruction.md": "56c8ecdd0084a07f8c97986c43af5c37ed03dd5aa994da6735cf59806162d1b2", + "solution/solve.sh": "3da9dbf589656512cd7a7004dc3752d3552a6af939cfe59eff79f91695fa42c7", + "task.toml": "b90a47ec26c1191d3c523cc094ea96b170e42fbfe176ef1f814c57d05390c097", + "tests/reference_illum1.png": "bb084f430bbc9d5b623a656920670dc06cee6d9132a9d550be4a1b99e41ab9e0", + "tests/test.sh": "928d1497ad21aef0c456294e3784f1fddf8493410f23584da89f9e02ec6bd134", + "tests/test_outputs.py": "64d6ce107503714ada346e5606832ee886be9950971cf81523e73d8ba28e8d37" + } + }, + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88", + "selection_rationale": "Legacy-language migration and behavioral compatibility", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "software-engineering", + "declared_difficulty": "easy", + "docker_image": "alexgshaw/cobol-modernization:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "c89318cf59efcc7040e3d946022e2616161b6733931a080cdeea7b1918e7ecc4", + "environment/Dockerfile": "8b175c83fc03ec7d16d6101516bc5100230b592f480e435279f4fe051c9280b2", + "environment/data/ACCOUNTS.DAT": "46f63eb29442ebc6716bcd8a0c36f4367910f2783f6b84a307ed9a61ccd1046e", + "environment/data/BOOKS.DAT": "db04da646207cd87f8e9c5b6f0b57d95953899f3eef22de1a81423f06ef4a9fd", + "environment/data/TRANSACTIONS.DAT": "1fa4cc5420ded94d585c563e8196b85166ebbae2bb8924b20014717fb2fb9ee3", + "environment/src/INPUT.DAT": "52fc984f0cf14af1495285d7efe02b9bae13159eea59255560ee938c4f78afee", + "environment/src/program.cbl": "27cb0e56f65f52fa75f346720b37a180b8e034ebe7d7622b979f195087bd6ebb", + "instruction.md": "a87074f1141b7345394cd88053d0debb0dc8f7de95308882c6538ee9a3b359da", + "solution/solve.sh": "7d4a0504e1686c1790bfbc074d4e598f4c631f54c2bc5f3d5fd786574b4810b9", + "task.toml": "4b620211dc3d6b1b989f0210c5ecd2f606974581c8bce27bd78659d48cf0cfbd", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "009180a7092583fc8143bb3ffc3957923aa0d61d0aac3a471f76eeb4aae54a88" + } + }, + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e", + "selection_rationale": "Native memory management debugging", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 1800, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "category": "debugging", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/custom-memory-heap-crash:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "a06ed23a2c467aa36d0d4a88e5c020b1076cc25b8d483517239658e9c596dde9", + "environment/Dockerfile": "6ee81e5a4fa2540443d7f27d577ba840728fc6aab9f301ab7919103673c78080", + "environment/patches/locale_classes.tcc.patch": "16951f4a4df6bf253886ac75f5a39fdbc328e415194493f63cb7589a364aa817", + "environment/patches/locale_init.cc.patch": "e97f133ceebd86f8881f972e0e45b0ec0a13a9bfa428d8032008707683d54292", + "environment/program/main.cpp": "b94538c76887cc2b2ab739b5ac4731cddd0faa8f4a9892a6265c062bbbeede14", + "environment/program/user.cpp": "92b203786fed1bf0b3aa1675c9cd2d82b2651170675064da8ffee73b98ef14f9", + "environment/program/user.h": "f43d155995a56c22b9cd1417daef7c3b5a976cdc662a98eef0feb438ae1eee77", + "instruction.md": "53c68ae14a3c3af9a97c40bfd9bdc30a5d98b37eaa2f69f763475745f70f108d", + "solution/solve.sh": "b0b1d10bea1cd5be14a82b35d522df616df1e67028f90d2a6bef90aaa98511a1", + "task.toml": "ed474be1983e212913aa8866ebb66216acbe36aef2339ac375d232aa46617481", + "tests/test.sh": "70a2b55f0c5757f6b123a216dfd202be4a646680bced28dddcd31fd8959a058f", + "tests/test_outputs.py": "4e1cb33f777fa7fb5e1536f9bea9eace6deccd515c67d4f87570adbe692914ba" + } + }, + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71", + "selection_rationale": "Security bug repair and behavior preservation", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "security", + "declared_difficulty": "hard", + "docker_image": "alexgshaw/fix-code-vulnerability:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "87f28dbad183c3a634887696d328ca68a8a0858f132bc48cb7341c22267d38f7", + "environment/Dockerfile": "0917470986224df4daf4854a71302ae81c73ba5dd6e25be1232d29b758a5b3ab", + "instruction.md": "89b1abdf0af19399f720233b5cdf6b2e8e0ff7b6e3fe3e2a3af0d4d298a8641a", + "solution/solve.sh": "7a0131b369d83ae8546ecc3f9b4e23d38240f1bd1f0941e4430fa85b5182ee03", + "task.toml": "9aa2bd14ce9408289d199d0608bc3fa65b8b5e23374c9641020de62197dbf1e2", + "tests/test.sh": "f1611da693cb4cfc5e8482698c6a1858a5c67220ae9db89b50aed66584e9b17f", + "tests/test_outputs.py": "c006a5de95b69bd78055aa26cc055734dbd50718ede25d331f767bfee4ef7f26" + } + }, + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097", + "selection_rationale": "Service configuration and observable request behavior", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "system-administration", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/nginx-request-logging:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d79997586d1c14ed8dbfea050e3580b25500e04c8345acfd3f294152433c795a", + "environment/Dockerfile": "43003466d89ab83484a3fa618e35b58cdb74a002b1e8716ef30f12b3ac8302d8", + "instruction.md": "26a7ac98aced6107f147206790ada77bbf8ed3c25fb360cf323d5fc6889edf99", + "solution/solve.sh": "2454bbbf7c201a9a778165f286373ba2cb991a0eb1d026a8005d57ae71250dd9", + "task.toml": "b7a78d00be13656f75069197100b7b8c4fd268decf6f78929c853b3222a2729d", + "tests/test.sh": "27761195f53cbfe5941fe39b573631f6c926869e64b7c43220377e1e116528e7", + "tests/test_outputs.py": "045cc716c14efde3b0dcff5fc7c85ec5d18bfc6ce66f8b40a418fa2a3a4acda0" + } + }, + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa", + "selection_rationale": "Cross-branch repository operations", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "system-administration", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/git-multibranch:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "3b9473429381d9eacabab498cb4a8915aa2bca24fa857ef862e5991d3f81f113", + "environment/Dockerfile": "88d9a9f4fef1ae7fddf632799dedf5a2fb6489460a93b610c33dd8b122982821", + "environment/default.conf": "a3a4c5d0ac9cfe79b0510e4225c80fa3988c27fbac13622505886bf66449e04a", + "instruction.md": "54014a38d7526d65faeb6c11d9b7ed3d4a5cb5d973d68d3da3b68b8c0289e125", + "solution/solve.sh": "300ca96b79db8d77f14742109aaf76048aafc4644ced750b1bc5c2761e8d0175", + "task.toml": "4b9981604578a31205257cca36a28d0c89c9d8d1058541330acd11d3f8d65f82", + "tests/test.sh": "ddbde3eea6ffa8fda105cbd86597a53bd409f41e13223900aa27e77a84b6b155", + "tests/test_outputs.py": "f03c6867663b8e8fb7899df51ee4df26411fab62ed491753964b801e591d16c5" + } + }, + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4", + "selection_rationale": "Database recovery and data integrity", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "file-operations", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/db-wal-recovery:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "600762031a1a02ecc3b46d70f7683f3ff930b47c8a3d0e27cc31b146158c3126", + "environment/Dockerfile": "63a260531c62f4ae72060db17f295c0aa03cfc6321f8162c324b7aa6b3f81302", + "environment/main.db": "68cb288eb29e76d814b4e7ee6887b37c415675e644f88173bc191348e290d119", + "environment/main.db-wal.encrypted": "68753bdce26196a53194260d001544cc6bb8f9f38e2f2f6826d09798e31e6e26", + "instruction.md": "5a20375162921b8911c67035d1905df7fd335d1bed03044271e608b2c3c93b26", + "solution/solve.sh": "ed8530c94103bf19783a1d41bc63fe4db478cdd9629f22fee25c45a45699f5e8", + "task.toml": "1cdc2c615e259bc10265efbe71b518332070c1c002e226566399e9788f18d0aa", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "88dc5c2a3abba75bca0e0f9136223ae5bb585d5984b8d9eba51a7829c1d2ba49" + } + }, + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49", + "selection_rationale": "Data integration and cross-input consistency", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "data-processing", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/multi-source-data-merger:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9bf37162a4c1b88ed93627830bb8512ebae154325d7cb069c883144a0e4c69d6", + "environment/Dockerfile": "c95a40d3d35b473dc15ae1e8784ba7fb1725805c0583fc574dd2764325d75e4f", + "environment/data/source_a/users.json": "75e51e2d3ab0bdba2269d7cf3757576f84f3ee2c13407d2db94289a5958a1778", + "environment/data/source_b/users.csv": "8e87d16dc47d166edd8e56165955bad265b630d53d02762367c816affb5fa760", + "environment/data/source_c/users.parquet": "23d1ea51c4170bdba53febb885a0f495e28de525a52667347b2932136b9a72fe", + "instruction.md": "d1d28bb6e37f7a255a1b7f78aae8591f35c0494f33e4913392424ad954aab944", + "solution/solve.sh": "1b3d0d50e31930d042495d48f4c8e95f1a9c1a979c803b7f7cec651322537e1e", + "task.toml": "ee3964c0ee040f0a0cb121cefefbc15443d127684acb003fa5b61dd42f31c694", + "tests/test.sh": "d5a00e64413c0e1ae452f38a1aa1a47c3ed7b4416bbfccae1a4c053025a336d1", + "tests/test_outputs.py": "a070888930cf829aeca039771260b1386bda990ea805f313468d381ba3608ca5" + } + }, + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4", + "selection_rationale": "Database query performance and correctness", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 1800.0, + "category": "data-science", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/query-optimize:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "a82c53e333a0ac0b0b410b5e26cb8e7305dece31db84d340248ffb13d5a45160", + "environment/Dockerfile": "064818771f200e0517d22068f87229a7a95d0a82b0778c84ca7203bf1926a0b8", + "environment/task-deps/my-sql-query.sql": "43382792f0f438625d643e34abe144febd5669518f4ab1ab550a15c6d9314a21", + "instruction.md": "473e12d320d2732f679fb631c2a4dab4a01deea6775686718ae0c417b8a39447", + "solution/solve.sh": "0d95f3cedb287ff9d47c796cbaf812c300d1222c10fa80c49e8acb0e81eb6978", + "task.toml": "6e88f2a86df699594bcb1bf81df4ed694505f79c9420f671335b02524bbe3868", + "tests/golden.sql": "4da1885d56b3ffa7b7b22b422996d018795611f6963499d551582881b74b84fd", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "2068ae80c57af8d52a7ae510f0666732c057806e556b480970f36f14bae683f6" + } + }, + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de", + "selection_rationale": "Numerical sampling and statistical invariants", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "scientific-computing", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/adaptive-rejection-sampler:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "95408217b9c484ffc18a02a91b7706323b6902919c72b24866a32e27a23543e2", + "environment/Dockerfile": "e644a3343e339303230f1441b7383f076d21c69a5a7aa6430b4f8a9d27f15c8b", + "environment/protected.tar.gz.enc": "7fc9bf1347bb97da8dede04b2c58638f225070913e222d76627068aebd1b1df2", + "instruction.md": "e6d28c28acd76bef61817427c74c2a29e3e8d75f535cbd708afcbfd02c54013f", + "solution/solve.sh": "fe443a8082b5183ba1623311ba0ee958888aefa633eff93766b9ae14e9bdd84d", + "task.toml": "6ed7b37f74beb46e9d47b0f5fad53025667c9548baff4e96d467b56d429fab1a", + "tests/test.sh": "38b43560d173cc2b952c3a3e17b8a480216d84e33450515e047bcb0d806b1e0a", + "tests/test_outputs.py": "547dc6e107f034f41703722aeceb6d0236e3fb69116fc3f2fbaba11884de352f" + } + }, + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53", + "selection_rationale": "Constrained algorithmic planning", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 1200, + "work_budget_seconds": 1020, + "verifier_timeout_seconds": 1200.0, + "category": "personal-assistant", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/constraints-scheduling:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d54a690970149e1216bd4fc066b45069c7948c13e3b1ef2a213a74342805f209", + "environment/Dockerfile": "7d9aba072a4a907229728b9739c8cf821fc2c98450babd64452682caf3759c84", + "environment/inputs/alice_calendar.ics": "5a5d45804bffd339c2479d49e76fe817b38244d14d389afe1de583443808ecbc", + "environment/inputs/bob_calendar.ics": "c97e85dbcdd756758f8a80f4db5fd5c1dcacee79298e4d2262f46de5ff680b50", + "environment/inputs/carol_calendar.ics": "32a0f8687637df0a12b8e5c56d49c4192bc3e790856943632d70f237c7c1363b", + "instruction.md": "342def78177f9bc41bea4d11af679bc385a9dda2ac8a1e425be384a5b7f937b6", + "solution/solve.sh": "802d81ec064fad3050bbade95a2f847ac24e0b7f80fa69b0bcee0347dc92fe77", + "task.toml": "7b9842de16d74e8faf8c7c0aa03b0b81145941cb12dd322d81ae23fc48607250", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "1422b345ba3aacdfc4184e962811b6379988435896c50d8cf62903e70990c5d1" + } + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79", + "selection_rationale": "Model packaging and command-line contracts", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "model-training", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/pytorch-model-cli:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "35a093fa5f1592737784ba718c2e3e9ae17ff8b5e53e003fae560ddce091039c", + "environment/Dockerfile": "8fd5c0f21a6914cd440d10c84c9db06fec606f20dc3e0363b17a3b8c220dc4d9", + "environment/task-deps/cJSON.c": "0e7995261640c96db16ed2c91a5d981efa917cf5e72d0f97ec45a1179ae3ad73", + "environment/task-deps/cJSON.h": "0909160541bcc510e0b2d8c7a9cc051fa1459a0f13be4d8e56346a15491f2f2f", + "environment/task-deps/image.png": "f4d5dfefe463939996541404340e7099d5957b56e1df3760d13b5d2de754db53", + "environment/task-deps/lodepng.cpp": "67873c5f6272f779a143a9c81070e8783327c5552d0b9de2aaee2cb779308917", + "environment/task-deps/lodepng.h": "b735b7e0357319973e9488d520d7f270d734a6e9e1bf924fb1d5b8d26d7e97ce", + "environment/task-deps/model.pth": "a32103eef09936e586b1a7dc7e64d279500a38ff30fdd6df2167e28eb954eee6", + "environment/task-deps/model.py": "e4d3a5c728449656d0efde7871ee227619348352769795ed1be1a8b7f835fddc", + "instruction.md": "ce36ee1f6c589f81ee6a6be4967e25c76a2ed697006928023f42dfb9f28a6750", + "solution/solve.sh": "3b34e561c9ef0a53c9948655847846eb4811aaac67b990480aa6aac1b04bfe66", + "task.toml": "fc1c018efbd5d0537ac0741d53d1ebad3886faa51d14cbde8ec953856411bc0d", + "tests/test.sh": "43fb67e537c7adc2df11c2367f47c876cb5d16a7c1ae16d0339c3a3364bb2b06", + "tests/test_outputs.py": "370702c57a4e0c85f70ee37a5c357ae3a5df8480a275bb6263b0376f2f6c093b" + } + } + ], + "total_model_runs": 12, + "max_concurrent_trials": 2, + "whole_trial_retries": 0, + "preflight": [ + { + "arm": "main", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/install-only.json", + "config_sha256": "b8fc1d1d825cc514c345026bdfce4cd867443125aac0de1d0d6e02b6c3e7c30a", + "job_name": "dev12-main-install-only-20261011" + } + ], + "queue": [ + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/cobol-modernization", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/cobol-modernization.json", + "job_name": "dev12-cobol-modernization-baseline-20261011", + "config_sha256": "2680126c2f162d58948cd30565806bf6cb1f026d561c87b942ba4735173ea6a2" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/fix-code-vulnerability", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/fix-code-vulnerability.json", + "job_name": "dev12-fix-code-vulnerability-baseline-20261011", + "config_sha256": "4b2de43b547ddd92d748f7658c44b98b4fef25efd406710ace6d1713b9c789a7" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/nginx-request-logging", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/nginx-request-logging.json", + "job_name": "dev12-nginx-request-logging-baseline-20261011", + "config_sha256": "4a79d4fc4431acd942548f36188c1820ac5406fc6a1385cf505daf02aa190da3" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/git-multibranch", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/git-multibranch.json", + "job_name": "dev12-git-multibranch-baseline-20261011", + "config_sha256": "ae9c6d5830c9dcb8aab8b1356441b47731957e2ad5e5747327b7caa1a6c037a1" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/db-wal-recovery", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/db-wal-recovery.json", + "job_name": "dev12-db-wal-recovery-baseline-20261011", + "config_sha256": "475b7df31549e51f04f4f9171332da5a8ba03f7fbec2b3788fbf897340c0c91b" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/multi-source-data-merger", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/multi-source-data-merger.json", + "job_name": "dev12-multi-source-data-merger-baseline-20261011", + "config_sha256": "5eac4294149a11f037f732a32e1a8874613d8a28884843955481a9e8ef10b7bd" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/query-optimize", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/query-optimize.json", + "job_name": "dev12-query-optimize-baseline-20261011", + "config_sha256": "29ad0478282cd6fb86a1b209a93ffe33eee724c321916732db001b7beef8ea7e" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/adaptive-rejection-sampler", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/adaptive-rejection-sampler.json", + "job_name": "dev12-adaptive-rejection-sampler-baseline-20261011", + "config_sha256": "e0fec8e3f1f8e077a5384058ea5b6e4733973a7475b02556f90cfc0f4ba96f6d" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/pytorch-model-cli", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/pytorch-model-cli.json", + "job_name": "dev12-pytorch-model-cli-baseline-20261011", + "config_sha256": "bc915a7de1286954473271e2232f9c83af983c23b885d099316b33dc3c1d445d" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/constraints-scheduling", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/constraints-scheduling.json", + "job_name": "dev12-constraints-scheduling-baseline-20261011", + "config_sha256": "33813c2fd5b45efd3b7bf8a765fcb1f74c702821a3d4f473e4b479200433734e" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/custom-memory-heap-crash", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/custom-memory-heap-crash.json", + "job_name": "dev12-custom-memory-heap-crash-baseline-20261011", + "config_sha256": "4678ab6961d82a2d597911bd8d4a53040a4be526c1e97afb27c614ecedb1c05f" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/build-pov-ray", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/build-pov-ray.json", + "job_name": "dev12-build-pov-ray-baseline-20261011", + "config_sha256": "42e1bb7d2a1a6d5cf9f059a121140f9135d8b4632482a00d767a9aba2e1ac2e7" + } + ], + "log_dir": "jobs/dev12-baseline-20261011-orchestration", + "budget_policy": "Native task agent timeout minus 180 seconds for runtime work; setup separately allows 1800 seconds. Native verifier limits remain unchanged. Build POV-Ray retains its unusually long 12000-second native limit (11820-second model work).", + "execution_policy": "One run per selected task. Run all twelve installation preflights before models. Queue is fixed by ascending native work budget, stable within ties; run consecutive pairs with at most two concurrent model trials.", + "failure_policy": "Retain every selected trial including setup failures, verifier errors and scored failures; no outcome-based substitutions or model reruns. Only verifier timeouts may retry, at most three total attempts on the same output with the native per-attempt limit.", + "preflight_policy": "All selected verifier dependency profiles are not_configured. Agent installation success does not certify verifier dependencies. Registered profiles, if present, must pass before model work.", + "information_policy": "Selection used task names before downloading. Only public metadata/instructions may be inspected before every agent output freezes; task files are hashed as bytes without inspecting hidden test or solution contents. Audit external references afterwards.", + "scope": "Development baseline for twelve new tasks, not a representative full-dataset score or a pass-rate improvement experiment. Existing 22 task records remain separate; remaining 55 tasks are not opened." +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/fix-code-vulnerability.json b/benchmarks/harbor/configs/dev12-baseline-20261011/fix-code-vulnerability.json new file mode 100644 index 0000000..9e19897 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/fix-code-vulnerability.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/git-multibranch.json b/benchmarks/harbor/configs/dev12-baseline-20261011/git-multibranch.json new file mode 100644 index 0000000..5956b98 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/git-multibranch.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/install-only.json b/benchmarks/harbor/configs/dev12-baseline-20261011/install-only.json new file mode 100644 index 0000000..04d81dc --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/install-only.json @@ -0,0 +1,66 @@ +{ + "n_concurrent_trials": 2, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb" + }, + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88" + }, + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e" + }, + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71" + }, + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097" + }, + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa" + }, + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4" + }, + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49" + }, + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4" + }, + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de" + }, + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53" + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/main-prompt.txt b/benchmarks/harbor/configs/dev12-baseline-20261011/main-prompt.txt new file mode 100644 index 0000000..56116f5 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/main-prompt.txt @@ -0,0 +1,9 @@ +You are nanoPyCodeAgent, a coding agent running non-interactively on a task handed to you up front. There is no user to reply to you: never ask a clarifying question, never stop to wait for confirmation, and never present a plan for approval — decide on your own and carry it out. Work the task through to the end, then check the result with the tools instead of assuming it worked. When it is done, answer with a short summary and no further tool calls: that reply is what ends the run. Runtime budget reminders report remaining time and model replies; when either is running low, prioritize saving the required deliverables, necessary verification, and a final summary. Do not start optional work after the requirements are satisfied. + +Identify the required deliverables, interfaces, and constraints. Treat examples as evidence, not as the full contract unless the task says so. Inspect relevant files, dependencies, and existing checks instead of guessing. Task-supplied workspace files, including source, executables, and tests, are available references unless the task restricts their use. Consult general documentation as needed. Do not seek or use task-specific reference solutions or hidden evaluation tests from outside the supplied materials unless the task explicitly requests them. Derive checks from the stated requirements. + +Verify the requested behavior through the real entry point when feasible. Check the content and meaning of outputs, not only command success or file existence. For logic or numerical work, choose a few representative and boundary cases justified by the task, using an independent reference, invariant, or consistency check when available. Do not derive every expectation from the implementation being tested or weaken requirements to make a check pass. Keep verification proportional to the task and remaining budget. + +When a check fails, inspect the error and relevant output, revise the underlying assumption or implementation, and rerun the affected check. Keep the task's constraints intact. Once appropriate checks support the required behavior, save the deliverables and finish. State what was verified and any remaining limitation without claiming unobserved success. + +Prefer the read tool for viewing files, the edit tool for changing part of an existing file, and the write tool for creating files or rewriting them whole. Use the bash tool to run commands, search with grep, and complete tasks that need real command output instead of guessing. \ No newline at end of file diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/multi-source-data-merger.json b/benchmarks/harbor/configs/dev12-baseline-20261011/multi-source-data-merger.json new file mode 100644 index 0000000..6c55a0a --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/multi-source-data-merger.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/nginx-request-logging.json b/benchmarks/harbor/configs/dev12-baseline-20261011/nginx-request-logging.json new file mode 100644 index 0000000..be8ea55 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/nginx-request-logging.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/pytorch-model-cli.json b/benchmarks/harbor/configs/dev12-baseline-20261011/pytorch-model-cli.json new file mode 100644 index 0000000..0b76cb3 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/pytorch-model-cli.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/query-optimize.json b/benchmarks/harbor/configs/dev12-baseline-20261011/query-optimize.json new file mode 100644 index 0000000..35c2998 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/query-optimize.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/selection.json b/benchmarks/harbor/configs/dev12-baseline-20261011/selection.json new file mode 100644 index 0000000..655c187 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/selection.json @@ -0,0 +1,96 @@ +{ + "schema_version": 1, + "selected_at": "2026-10-11T14:32:49.067833+00:00", + "main_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "dataset_name": "terminal-bench/terminal-bench-2-1", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "catalog_sha256": "7fc7d95a2077ede76c4f47890a09de93c2fe558252a122ef830288ad293ceaaa", + "selection_method": "Purposive coverage of twelve public task-name categories before downloading task instructions or seeing any baseline outcomes. This is a development cohort, not a random representative benchmark sample. Retain all selected tasks and every failure; do not replace based on outcomes.", + "prior_experiment_task_names": [ + "terminal-bench/build-cython-ext", + "terminal-bench/caffe-cifar-10", + "terminal-bench/cancel-async-tasks", + "terminal-bench/circuit-fibsqrt", + "terminal-bench/dna-assembly", + "terminal-bench/kv-store-grpc", + "terminal-bench/llm-inference-batching-scheduler", + "terminal-bench/log-summary-date-ranges", + "terminal-bench/merge-diff-arc-agi-task", + "terminal-bench/model-extraction-relu-logits", + "terminal-bench/mteb-leaderboard", + "terminal-bench/openssl-selfsigned-cert", + "terminal-bench/path-tracing", + "terminal-bench/pypi-server", + "terminal-bench/pytorch-model-recovery", + "terminal-bench/qemu-alpine-ssh", + "terminal-bench/regex-chess", + "terminal-bench/regex-log", + "terminal-bench/schemelike-metacircular-eval", + "terminal-bench/torch-pipeline-parallelism", + "terminal-bench/torch-tensor-parallelism", + "terminal-bench/write-compressor" + ], + "tasks": [ + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb", + "selection_rationale": "Native compilation and build tooling" + }, + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88", + "selection_rationale": "Legacy-language migration and behavioral compatibility" + }, + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e", + "selection_rationale": "Native memory management debugging" + }, + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71", + "selection_rationale": "Security bug repair and behavior preservation" + }, + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097", + "selection_rationale": "Service configuration and observable request behavior" + }, + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa", + "selection_rationale": "Cross-branch repository operations" + }, + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4", + "selection_rationale": "Database recovery and data integrity" + }, + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49", + "selection_rationale": "Data integration and cross-input consistency" + }, + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4", + "selection_rationale": "Database query performance and correctness" + }, + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de", + "selection_rationale": "Numerical sampling and statistical invariants" + }, + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53", + "selection_rationale": "Constrained algorithmic planning" + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79", + "selection_rationale": "Model packaging and command-line contracts" + } + ], + "instruction_or_hidden_test_review_before_selection": false +} diff --git a/benchmarks/harbor/scripts/run_comparison.py b/benchmarks/harbor/scripts/run_comparison.py new file mode 100644 index 0000000..9266f1f --- /dev/null +++ b/benchmarks/harbor/scripts/run_comparison.py @@ -0,0 +1,155 @@ +"""Run a frozen comparison plan through the standard Harbor entry point. + +This orchestrates existing repository worktrees; it does not install dependencies, +modify task environments, implement an adapter, or retry a trial. +""" + +import argparse +import ast +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import subprocess + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def prompt_from_source(path): + values = {} + for node in ast.parse(path.read_text()).body: + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name) and target.id in { + "_TOOL_GUIDANCE", "HEADLESS_SYSTEM_PROMPT", + }: + values[target.id] = eval( + compile(ast.Expression(node.value), str(path), "eval"), + {"__builtins__": {}}, values, + ) + return values["HEADLESS_SYSTEM_PROMPT"] + + +def validate_plan(root, plan_path, plan): + jobs = plan["preflight"] + plan["queue"] + names = [job["job_name"] for job in jobs] + if len(names) != len(set(names)): + raise ValueError("Job names must be unique") + if len(plan["queue"]) != plan["total_model_runs"]: + raise ValueError("Model run count does not match the registered queue") + if plan["max_concurrent_trials"] != 2 or plan["whole_trial_retries"] != 0: + raise ValueError("This runner requires concurrency two and no trial retries") + for arm in plan["arms"].values(): + source = root / arm["source_root"] + actual_ref = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=source, text=True, + ).strip() + if actual_ref != arm["source_ref"]: + raise ValueError(f"Source revision changed: {source}") + subprocess.run( + ["git", "diff", "--exit-code", "HEAD", "--", "src", "benchmarks/harbor/src"], + cwd=source, check=True, stdout=subprocess.DEVNULL, + ) + expected = plan_path.parent / arm["prompt_file"] + if digest(expected) != arm["prompt_sha256"]: + raise ValueError(f"Prompt file changed: {expected}") + if prompt_from_source(source / "src/nanopycodeagent/agent.py") != expected.read_text(): + raise ValueError(f"Source prompt does not match the plan: {source}") + for task in plan["tasks"]: + cache = Path.home() / ".cache/harbor/tasks/packages" / task["name"] / task["ref"].split(":")[1] + for name, expected in task["task_file_sha256"].items(): + if digest(cache / name) != expected: + raise ValueError(f"Task file changed: {cache / name}") + for job in jobs: + config_path = root / job["config"] + if digest(config_path) != job["config_sha256"]: + raise ValueError(f"Configuration changed: {config_path}") + config = json.loads(config_path.read_text()) + limit = 1 if job in plan["queue"] else 2 + if config["n_concurrent_trials"] > limit or config["n_attempts"] != 1: + raise ValueError("Per-job concurrency or attempts exceed the plan") + + +def run_job(root, plan, job, record, install_only): + status_path = record / (job["job_name"] + ".json") + if status_path.exists(): + previous = json.loads(status_path.read_text()) + if previous["status"] == "finished": + return previous + raise RuntimeError(f"Previously started job requires inspection; refusing to retry: {status_path}") + source = root / plan["arms"][job["arm"]]["source_root"] + command = [ + "uv", "run", "--project", "benchmarks/harbor", "python", "-m", "harbor_adapter", + "run", "--config", str(root / job["config"]), "--job-name", job["job_name"], + "--cache-dir", str(root / ".cache/nanopy-harbor"), + ] + if install_only: + command.append("--install-only") + summary_path = source / "jobs" / (job["job_name"] + "-input") / "summary.json" + state = { + **job, "status": "started", "started_at": datetime.now(timezone.utc).isoformat(), + "command": command, "cwd": str(source), "summary_path": str(summary_path), + } + status_path.write_text(json.dumps(state, indent=2) + "\n") + with (record / (job["job_name"] + ".log")).open("w") as log: + code = subprocess.run(command, cwd=source, stdout=log, stderr=subprocess.STDOUT).returncode + state.update(status="finished", exit_code=code, finished_at=datetime.now(timezone.utc).isoformat()) + if summary_path.exists(): + state["summary"] = json.loads(summary_path.read_text()) + status_path.write_text(json.dumps(state, indent=2) + "\n") + print(json.dumps({"job": job["job_name"], "exit_code": code, "status": state["status"]}), flush=True) + return state + + +def preflight_passed(states, expected_counts): + return bool(states) and len(states) == len(expected_counts) and all( + s.get("exit_code") == 0 + and s.get("summary", {}).get("trials") + and len(s["summary"]["trials"]) == expected + and all(t["status"] == "setup_passed" for t in s["summary"]["trials"]) + for s, expected in zip(states, expected_counts) + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--plan", type=Path, required=True) + parser.add_argument("--phase", choices=["preflight", "model"], required=True) + args = parser.parse_args() + root, plan_path = Path.cwd(), args.plan.resolve() + plan = json.loads(plan_path.read_text()) + validate_plan(root, plan_path, plan) + record = root / plan["log_dir"] + record.mkdir(parents=True, exist_ok=True) + record.chmod(0o700) + registration = record / "registration.json" + current = {"plan_sha256": digest(plan_path), "plan": plan} + if registration.exists(): + if json.loads(registration.read_text()) != current: + raise ValueError("The registered plan is immutable") + else: + registration.write_text(json.dumps(current, indent=2) + "\n") + expected_counts = [ + len(json.loads((root / job["config"]).read_text())["tasks"]) + for job in plan["preflight"] + ] + if args.phase == "preflight": + states = [run_job(root, plan, j, record, True) for j in plan["preflight"]] + return 0 if preflight_passed(states, expected_counts) else 1 + preflight = [json.loads((record / (j["job_name"] + ".json")).read_text()) for j in plan["preflight"]] + if not preflight_passed(preflight, expected_counts): + raise RuntimeError("Install-only checks must finish successfully before model work") + states = [] + with ThreadPoolExecutor(max_workers=2) as pool: + for offset in range(0, len(plan["queue"]), 2): + batch = plan["queue"][offset:offset + 2] + futures = [pool.submit(run_job, root, plan, job, record, False) for job in batch] + states.extend(f.result() for f in futures) + return int(any(s["exit_code"] != 0 for s in states)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/tests/test_comparison.py b/benchmarks/harbor/tests/test_comparison.py new file mode 100644 index 0000000..61faa56 --- /dev/null +++ b/benchmarks/harbor/tests/test_comparison.py @@ -0,0 +1,57 @@ +"""Comparison orchestration must preserve failed trials and preflight gates.""" + +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +spec = importlib.util.spec_from_file_location( + "run_comparison", Path(__file__).parents[1] / "scripts/run_comparison.py", +) +comparison = importlib.util.module_from_spec(spec) +spec.loader.exec_module(comparison) + + +@pytest.mark.parametrize("states", [ + [], [{"exit_code": 0}], + [{"exit_code": 0, "summary": {"trials": []}}], + [{"exit_code": 1, "summary": {"trials": [{"status": "setup_passed"}]}}], + [{"exit_code": 0, "summary": {"trials": [{"status": "setup_failed"}]}}], +]) +def test_incomplete_or_failed_preflight_cannot_start_models(states): + assert not comparison.preflight_passed(states, [1] * len(states)) + + +def test_successful_installation_does_not_require_a_registered_profile(): + state = {"exit_code": 0, "summary": {"trials": [{ + "status": "setup_passed", "bootstrap": {"verifier_preflight": {"status": "not_configured"}}, + }]}} + assert comparison.preflight_passed([state], [1]) + assert not comparison.preflight_passed([state], [4]) + assert not comparison.preflight_passed([state], [1, 1]) + + +def test_failed_job_is_recorded_and_never_retried(tmp_path, monkeypatch): + plan = {"arms": {"control": {"source_root": "source"}}} + job = {"arm": "control", "config": "config.json", "job_name": "test-job"} + calls = [] + def run(command, **kwargs): + calls.append(command) + return SimpleNamespace(returncode=1) + monkeypatch.setattr(comparison.subprocess, "run", run) + first = comparison.run_job(tmp_path, plan, job, tmp_path, False) + second = comparison.run_job(tmp_path, plan, job, tmp_path, False) + assert first == second + assert first["exit_code"] == 1 + assert len(calls) == 1 + assert calls[0][:8] == ["uv", "run", "--project", "benchmarks/harbor", "python", "-m", "harbor_adapter", "run"] + + +def test_interrupted_job_cannot_silently_restart(tmp_path): + job = {"job_name": "interrupted"} + (tmp_path / "interrupted.json").write_text(json.dumps({"status": "started"})) + with pytest.raises(RuntimeError, match="refusing to retry"): + comparison.run_job(tmp_path, {}, job, tmp_path, False) From d7dbcec4fa29c191b80453e8d61f2e1ec4c8090d Mon Sep 17 00:00:00 2001 From: minixalpha Date: Sun, 11 Oct 2026 08:20:11 -0700 Subject: [PATCH 2/5] fix(benchmarks): cache original images with verified range downloads --- benchmarks/harbor/README.md | 27 +++ benchmarks/harbor/README.zh-CN.md | 22 ++ benchmarks/harbor/scripts/cache_image.py | 236 ++++++++++++++++++++ benchmarks/harbor/tests/test_image_cache.py | 154 +++++++++++++ 4 files changed, 439 insertions(+) create mode 100644 benchmarks/harbor/scripts/cache_image.py create mode 100644 benchmarks/harbor/tests/test_image_cache.py diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 3b841aa..056f454 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -89,6 +89,33 @@ when it runs; a successful preflight is not a guarantee against later outages. Full standard runs enable the timeout retry policy below by default, with zero whole-trial retries. CI checks these contracts without model calls or Docker. +### Cache original task images + +If Docker image pulls repeatedly stall, the optional cache utility downloads the +same public Docker Hub image in bounded parallel HTTP ranges: + +```bash +python benchmarks/harbor/scripts/cache_image.py \ + alexgshaw/git-multibranch:20251031 \ + --cache .cache/nanopy-harbor/images --load +``` + +The first request records the registry manifest digest; later requests reject a +changed tag. Every downloaded blob must match its size and SHA256, and each +uncompressed layer must match the original image configuration. The resulting +[OCI archive](https://github.com/opencontainers/image-spec/blob/main/image-layout.md) +preserves the original manifest, configuration, and compressed layers byte for +byte. Docker import must retain the original repository digest. + +This Linux utility supports public single-platform `linux/amd64` images with +gzip layers and SHA256 descriptors. `--load` requires Docker's containerd image +store. Use one cache writer at a time; `--workers` controls parallel range +requests (default eight, maximum sixteen). Completed blobs and partial ranges +can be reused after an interrupted download, with complete blobs revalidated. +Keep the manifest lock records with the run evidence. This only warms Docker's +image cache: continue using the standard benchmark entry point and installation +preflight, and retain any earlier environment-start failures separately. + ### Direct Harbor invocation For adapter development or comparison with published versions, the lower-level diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index 12ca806..0497dac 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -71,6 +71,28 @@ uv 0.9.5、Python 3.13、pytest 8.4.1 和 pytest-json-ctrf 0.3.5，不执行题 通过不能保证后续没有网络故障。完整标准运行默认启用下述验证超时重试，整题重试 次数为零。CI 会验证这些约定，不调用模型、不启动 Docker。 +### 缓存原始任务镜像 + +如果 Docker 拉取镜像反复停滞，可用缓存工具从同一 Docker Hub 公共仓库分段并行下载： + +```bash +python benchmarks/harbor/scripts/cache_image.py \ + alexgshaw/git-multibranch:20251031 \ + --cache .cache/nanopy-harbor/images --load +``` + +首次请求记录仓库的 manifest 摘要；后续请求若发现 tag 指向的摘要改变，会拒绝继续。 +每个下载对象都必须匹配大小和 SHA256，每层解压后的摘要也必须匹配原始镜像配置。 +生成的 [OCI 归档](https://github.com/opencontainers/image-spec/blob/main/image-layout.md) +逐字节保留原始 manifest、配置和压缩层；Docker 导入后必须保留原始仓库摘要。 + +该 Linux 工具支持使用 gzip 层和 SHA256 描述符的公共单平台 `linux/amd64` 镜像。 +`--load` 需要 Docker 的 containerd 镜像存储。同一缓存目录同时只允许一个写入进程； +`--workers` 控制分段请求并发数，默认 8、最多 16。下载中断后可复用已完成的对象和分段， +完整对象会重新校验。请将 manifest 锁定记录纳入实验的证据归档。 +这个步骤只准备 Docker 镜像缓存；随后仍需使用标准 benchmark 入口并执行安装预检， +之前出现的环境启动失败应单独保留。 + ### 直接调用 Harbor 开发 adapter 或对照已发布版本时，仍可使用底层 Harbor 入口，并固定容器内 agent 版本： diff --git a/benchmarks/harbor/scripts/cache_image.py b/benchmarks/harbor/scripts/cache_image.py new file mode 100644 index 0000000..062bb1d --- /dev/null +++ b/benchmarks/harbor/scripts/cache_image.py @@ -0,0 +1,236 @@ +"""Cache an unchanged public Docker Hub image through verified HTTP ranges. + +For slow or interrupted Docker pulls, create an OCI archive, then optionally +load it into Docker's containerd image store. No layers are edited or rebuilt. +Only single-platform Linux images with SHA256 and gzip layers are supported. +""" + +import argparse +from concurrent.futures import ThreadPoolExecutor +import fcntl +import gzip +import hashlib +from http.client import IncompleteRead +import io +import json +from pathlib import Path +import re +import subprocess +import tarfile +import time +from urllib.error import URLError +from urllib.parse import urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + + +MANIFEST_TYPES = ( + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.oci.image.manifest.v1+json", +) +LAYER_TYPES = { + "application/vnd.docker.image.rootfs.diff.tar.gzip", + "application/vnd.oci.image.layer.v1.tar+gzip", +} + + +class SafeRedirect(HTTPRedirectHandler): + def redirect_request(self, request, response, code, message, headers, url): + if urlsplit(url).scheme != "https": + raise ValueError("Registry redirects must retain HTTPS") + redirected = super().redirect_request(request, response, code, message, headers, url) + if urlsplit(url).netloc != urlsplit(request.full_url).netloc: + redirected.remove_header("Authorization") + return redirected + + +def get(url, headers=None, limit=4 * 1024 * 1024): + # Never include exception URLs: blob redirects may contain signed queries. + for attempt in range(3): + try: + with build_opener(SafeRedirect()).open(Request(url, headers=headers or {}), timeout=30) as r: + data = r.read(limit + 1) + if len(data) > limit: + raise ValueError("Response exceeds requested size") + return r.status, dict(r.headers.items()), data + except (URLError, TimeoutError, OSError, IncompleteRead) as error: + if attempt == 2: + raise RuntimeError(f"Registry request failed: {type(error).__name__}") from None + time.sleep(attempt + 1) + + +def sha(data): + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def blob_path(cache, descriptor): + digest = descriptor["digest"] + if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest): + raise ValueError("Only SHA256 descriptors are supported") + if not isinstance(descriptor["size"], int) or descriptor["size"] < 0: + raise ValueError("Invalid descriptor size") + return cache / "blobs/sha256" / digest.split(":")[1] + + +def verify(path, descriptor): + if path.stat().st_size != descriptor["size"]: + raise ValueError("Blob size mismatch") + with path.open("rb") as stream: + actual = "sha256:" + hashlib.file_digest(stream, "sha256").hexdigest() + if actual != descriptor["digest"]: + raise ValueError("Blob SHA256 mismatch") + + +def fetch_blob(cache, descriptor, url, headers, workers=8, chunk_size=1024 * 1024): + path = blob_path(cache, descriptor) + if path.exists(): + verify(path, descriptor) + return path + path.parent.mkdir(parents=True, exist_ok=True) + parts = cache / "parts" / path.name + parts.mkdir(parents=True, exist_ok=True) + size = descriptor["size"] + + def fetch(start): + end = min(start + chunk_size, size) - 1 + part = parts / str(start) + if part.exists() and part.stat().st_size == end - start + 1: + return part + for attempt in range(3): + status, response_headers, data = get( + url, {**headers, "Range": f"bytes={start}-{end}"}, end - start + 1, + ) + response_headers = {k.lower(): v for k, v in response_headers.items()} + complete = start == 0 and end == size - 1 and status == 200 + expected_range = f"bytes {start}-{end}/{size}" + if not complete and (status != 206 or response_headers.get("content-range") != expected_range): + raise ValueError("Server did not return the exact requested range") + if len(data) == end - start + 1: + break + else: + raise ValueError("Incomplete range response") + temporary = part.with_suffix(".tmp") + temporary.write_bytes(data) + temporary.replace(part) + return part + + with ThreadPoolExecutor(max_workers=workers) as pool: + downloaded = list(pool.map(fetch, range(0, size, chunk_size))) + temporary = path.with_suffix(".tmp") + with temporary.open("wb") as out: + for part in downloaded: + out.write(part.read_bytes()) + verify(temporary, descriptor) + temporary.replace(path) + for part in downloaded: + part.unlink() + parts.rmdir() + return path + + +def validate_layers(cache, manifest): + config = json.loads(blob_path(cache, manifest["config"]).read_bytes()) + if config.get("os") != "linux" or config.get("architecture") != "amd64": + raise ValueError("Only Linux amd64 images are supported") + diff_ids = config["rootfs"]["diff_ids"] + if config["rootfs"].get("type") != "layers": + raise ValueError("Unsupported root filesystem configuration") + if len(diff_ids) != len(manifest["layers"]): + raise ValueError("Layer count differs from image configuration") + for layer, expected in zip(manifest["layers"], diff_ids): + if layer["mediaType"] not in LAYER_TYPES: + raise ValueError("Only gzip layers are supported") + with gzip.open(blob_path(cache, layer), "rb") as stream: + actual = "sha256:" + hashlib.file_digest(stream, "sha256").hexdigest() + if actual != expected: + raise ValueError("Uncompressed layer SHA256 differs from image configuration") + return config + + +def write_archive(cache, image, raw_manifest, output): + manifest = json.loads(raw_manifest) + if manifest.get("schemaVersion") != 2 or manifest.get("mediaType") not in MANIFEST_TYPES: + raise ValueError("Only single-platform schema 2 image manifests are supported") + descriptors = [manifest["config"], *manifest["layers"]] + for descriptor in descriptors: + verify(blob_path(cache, descriptor), descriptor) + validate_layers(cache, manifest) + manifest_descriptor = {"mediaType": manifest["mediaType"], "size": len(raw_manifest), "digest": sha(raw_manifest)} + manifest_path = blob_path(cache, manifest_descriptor) + manifest_path.write_bytes(raw_manifest) + descriptors.append(manifest_descriptor) + index = {"schemaVersion": 2, "manifests": [{ + **manifest_descriptor, "annotations": { + "io.containerd.image.name": "docker.io/" + image, + "org.opencontainers.image.ref.name": image.rsplit(":", 1)[1], + }, + }]} + temporary = output.with_suffix(".tmp") + with tarfile.open(temporary, "w") as archive: + for name, data in ( + ("oci-layout", b'{"imageLayoutVersion":"1.0.0"}'), + ("index.json", json.dumps(index).encode()), + ): + entry = tarfile.TarInfo(name) + entry.size = len(data) + archive.addfile(entry, io.BytesIO(data)) + seen = set() + for descriptor in descriptors: + path = blob_path(cache, descriptor) + if path not in seen: + archive.add(path, arcname=str(path.relative_to(cache)), recursive=False) + seen.add(path) + temporary.replace(output) + + +def cache_image(image, cache, workers=8, load=False): + if not re.fullmatch(r"[a-z0-9_-]+/[a-z0-9._-]+:[A-Za-z0-9._-]+", image): + raise ValueError("Expected a public Docker Hub owner/repository:tag") + cache.mkdir(parents=True, exist_ok=True) + repo, tag = image.rsplit(":", 1) + _, _, raw_token = get("https://auth.docker.io/token?service=registry.docker.io&scope=repository:" + repo + ":pull") + headers = {"Authorization": "Bearer " + json.loads(raw_token)["token"], "Accept": ", ".join(MANIFEST_TYPES)} + base = "https://registry-1.docker.io/v2/" + repo + _, response_headers, raw = get(base + "/manifests/" + tag, headers) + response_headers = {k.lower(): v for k, v in response_headers.items()} + if response_headers.get("docker-content-digest") != sha(raw): + raise ValueError("Registry manifest digest mismatch") + manifest = json.loads(raw) + if manifest.get("mediaType") not in MANIFEST_TYPES or manifest.get("schemaVersion") != 2: + raise ValueError("Only single-platform schema 2 image manifests are supported") + lock = {"image": image, "manifest_digest": sha(raw), "config_digest": manifest["config"]["digest"]} + lock_path = cache / (image.replace("/", "_").replace(":", "_") + ".json") + if lock_path.exists() and json.loads(lock_path.read_text()) != lock: + raise ValueError("Image tag changed since the first cache request") + lock_path.write_text(json.dumps(lock, indent=2) + "\n") + print(json.dumps(lock), flush=True) + for descriptor in [manifest["config"], *manifest["layers"]]: + fetch_blob(cache, descriptor, base + "/blobs/" + descriptor["digest"], headers, workers) + print(json.dumps({"verified_blob": descriptor["digest"], "bytes": descriptor["size"]}), flush=True) + output = cache / (lock_path.stem + ".oci.tar") + write_archive(cache, image, raw, output) + if load: + driver = subprocess.check_output(["docker", "info", "--format", "{{json .DriverStatus}}"], text=True) + if "io.containerd.snapshotter.v1" not in driver: + raise ValueError("OCI import requires Docker's containerd image store") + subprocess.run(["docker", "load", "--quiet", "--input", str(output)], check=True) + installed = json.loads(subprocess.check_output(["docker", "image", "inspect", image], text=True))[0] + if not any(ref.endswith("@" + lock["manifest_digest"]) for ref in installed["RepoDigests"]): + raise ValueError("Imported image does not retain the original manifest digest") + print(json.dumps({"archive": str(output), "loaded": load, **lock}), flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("image") + parser.add_argument("--cache", type=Path, required=True) + parser.add_argument("--workers", type=int, choices=range(1, 17), default=8) + parser.add_argument("--load", action="store_true") + args = parser.parse_args() + args.cache.mkdir(parents=True, exist_ok=True) + with (args.cache / ".writer.lock").open("w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + cache_image(args.image, args.cache, args.workers, args.load) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/harbor/tests/test_image_cache.py b/benchmarks/harbor/tests/test_image_cache.py new file mode 100644 index 0000000..28de6bc --- /dev/null +++ b/benchmarks/harbor/tests/test_image_cache.py @@ -0,0 +1,154 @@ +"""Integrity checks for optional, content-preserving image cache imports.""" + +import gzip +import importlib.util +import io +import json +from pathlib import Path +import tarfile + +import pytest + + +spec = importlib.util.spec_from_file_location( + "cache_image", Path(__file__).parents[1] / "scripts/cache_image.py", +) +cache_image = importlib.util.module_from_spec(spec) +spec.loader.exec_module(cache_image) + + +def descriptor(data, media_type="application/octet-stream"): + return {"digest": cache_image.sha(data), "size": len(data), "mediaType": media_type} + + +def put(cache, data, media_type): + desc = descriptor(data, media_type) + path = cache_image.blob_path(cache, desc) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return desc + + +def fixture_manifest(cache, diff_id=None): + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode="w") as archive: + entry = tarfile.TarInfo("example.txt") + entry.size = 5 + archive.addfile(entry, io.BytesIO(b"hello")) + layer = stream.getvalue() + compressed = gzip.compress(layer) + config = json.dumps({ + "os": "linux", "architecture": "amd64", + "rootfs": {"type": "layers", "diff_ids": [diff_id or cache_image.sha(layer)]}, + }).encode() + return json.dumps({ + "schemaVersion": 2, "mediaType": cache_image.MANIFEST_TYPES[0], + "config": put(cache, config, "application/vnd.docker.container.image.v1+json"), + "layers": [put(cache, compressed, "application/vnd.docker.image.rootfs.diff.tar.gzip")], + }).encode() + + +def test_parallel_ranges_reconstruct_original_blob(tmp_path, monkeypatch): + payload = b"0123456789abcdefghij" + requests = [] + + def get(url, headers, limit): + requested = headers["Range"].removeprefix("bytes=") + start, end = map(int, requested.split("-")) + requests.append((start, end)) + return 206, {"Content-Range": f"bytes {start}-{end}/{len(payload)}"}, payload[start:end + 1] + + monkeypatch.setattr(cache_image, "get", get) + path = cache_image.fetch_blob(tmp_path, descriptor(payload), "unused", {}, workers=2, chunk_size=6) + assert path.read_bytes() == payload + assert sorted(requests) == [(0, 5), (6, 11), (12, 17), (18, 19)] + requests.clear() + cache_image.fetch_blob(tmp_path, descriptor(payload), "unused", {}) + assert requests == [] + + +@pytest.mark.parametrize("status,content_range,body", [ + (200, None, b"abc"), + (206, "bytes 3-5/6", b"abc"), + (206, "bytes 0-2/6", b"ab"), +]) +def test_reject_inexact_or_truncated_ranges(tmp_path, monkeypatch, status, content_range, body): + monkeypatch.setattr(cache_image, "get", lambda *args: (status, {"Content-Range": content_range}, body)) + with pytest.raises(ValueError): + cache_image.fetch_blob(tmp_path, descriptor(b"abcdef"), "unused", {}, chunk_size=3) + assert not cache_image.blob_path(tmp_path, descriptor(b"abcdef")).exists() + + +def test_reject_corrupt_complete_download(tmp_path, monkeypatch): + monkeypatch.setattr(cache_image, "get", lambda *args: (200, {}, b"abd")) + with pytest.raises(ValueError, match="SHA256 mismatch"): + cache_image.fetch_blob(tmp_path, descriptor(b"abc"), "unused", {}) + assert not cache_image.blob_path(tmp_path, descriptor(b"abc")).exists() + + +def test_retry_short_transport_response_before_caching(tmp_path, monkeypatch): + replies = iter([(200, {}, b"ab"), (200, {}, b"abc")]) + monkeypatch.setattr(cache_image, "get", lambda *args: next(replies)) + path = cache_image.fetch_blob(tmp_path, descriptor(b"abc"), "unused", {}) + assert path.read_bytes() == b"abc" + + +def test_revalidate_cached_bytes(tmp_path): + desc = put(tmp_path, b"abc", "application/octet-stream") + path = cache_image.blob_path(tmp_path, desc) + path.write_bytes(b"abd") + with pytest.raises(ValueError, match="SHA256 mismatch"): + cache_image.fetch_blob(tmp_path, desc, "unused", {}) + path.write_bytes(b"ab") + with pytest.raises(ValueError, match="size mismatch"): + cache_image.fetch_blob(tmp_path, desc, "unused", {}) + + +def test_archive_preserves_manifest_and_all_blobs(tmp_path): + raw = fixture_manifest(tmp_path) + output = tmp_path / "image.tar" + cache_image.write_archive(tmp_path, "example/task:version", raw, output) + with tarfile.open(output) as archive: + index = json.load(archive.extractfile("index.json")) + reference = index["manifests"][0] + assert reference["digest"] == cache_image.sha(raw) + assert reference["annotations"]["io.containerd.image.name"] == "docker.io/example/task:version" + assert archive.extractfile("blobs/sha256/" + reference["digest"].split(":")[1]).read() == raw + manifest = json.loads(raw) + for desc in [manifest["config"], *manifest["layers"]]: + data = archive.extractfile("blobs/sha256/" + desc["digest"].split(":")[1]).read() + assert cache_image.sha(data) == desc["digest"] + + +def test_reject_mismatched_uncompressed_layer(tmp_path): + raw = fixture_manifest(tmp_path, cache_image.sha(b"different layer")) + with pytest.raises(ValueError, match="Uncompressed layer SHA256"): + cache_image.write_archive(tmp_path, "example/task:version", raw, tmp_path / "image.tar") + assert not (tmp_path / "image.tar").exists() + + +@pytest.mark.parametrize("digest", ["sha256:../../bad", "md5:abcd", "sha256:ABCDEF"]) +def test_reject_unsafe_or_unsupported_blob_paths(tmp_path, digest): + with pytest.raises(ValueError, match="Only SHA256"): + cache_image.blob_path(tmp_path, {"digest": digest, "size": 1}) + + +def test_changed_tag_is_rejected_before_downloading_blobs(tmp_path, monkeypatch): + raw = fixture_manifest(tmp_path) + (tmp_path / "example_task_version.json").write_text(json.dumps({"manifest_digest": "old"})) + replies = iter([ + (200, {}, b'{"token":"anonymous-read-token"}'), + (200, {"Docker-Content-Digest": cache_image.sha(raw)}, raw), + ]) + monkeypatch.setattr(cache_image, "get", lambda *args: next(replies)) + with pytest.raises(ValueError, match="Image tag changed"): + cache_image.cache_image("example/task:version", tmp_path) + + +def test_redirect_does_not_forward_registry_authorization(): + request = cache_image.Request("https://registry-1.docker.io/blob", headers={"Authorization": "Bearer token", "Range": "bytes=0-2"}) + redirected = cache_image.SafeRedirect().redirect_request(request, None, 307, "", {}, "https://cdn.docker.com/blob") + assert not redirected.has_header("Authorization") + assert redirected.get_header("Range") == "bytes=0-2" + with pytest.raises(ValueError, match="HTTPS"): + cache_image.SafeRedirect().redirect_request(request, None, 307, "", {}, "http://cdn.docker.com/blob") From e3a53fb6dcf89adb269b2a950521ce6eb6d4bfaf Mon Sep 17 00:00:00 2001 From: minixalpha Date: Sun, 11 Oct 2026 08:32:44 -0700 Subject: [PATCH 3/5] test(benchmarks): retain failed preflight and register infrastructure retry --- .../configs/dev12-baseline-20261011/README.md | 20 +- .../experiment-v2.json | 519 ++++++++++++++++++ .../preflight-v1-evidence.json | 416 ++++++++++++++ 3 files changed, 951 insertions(+), 4 deletions(-) create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v2.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/preflight-v1-evidence.json diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/README.md b/benchmarks/harbor/configs/dev12-baseline-20261011/README.md index 1e3ebd8..d549a6e 100644 --- a/benchmarks/harbor/configs/dev12-baseline-20261011/README.md +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/README.md @@ -13,8 +13,20 @@ is included. The fixed dataset revision is `sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a`. [Selection](selection.json) records the existing 22 tasks, selected revisions and -rationales. The [registered plan](experiment.json) records runtime, prompt, -configuration and task-file hashes, work limits, preflight and queue order. +rationales. The [original registered plan](experiment.json) records runtime, +prompt, configuration and task-file hashes, work limits, preflight and queue +order. The active [pre-model revision](experiment-v2.json) repeats installation +preflight under a new job name and registration directory after infrastructure +failures; all model configurations, tasks, runtime, budgets and queue entries +remain identical. + +The [first preflight evidence](preflight-v1-evidence.json) retains five successful +installations, six image-pull/environment-start errors, and one agent setup +timeout while `apt-get update` was still downloading package indexes. No models +or official verifiers ran. All twelve original images are now cached, including +six downloaded with the verified image-cache utility. Their original registry +digests are recorded. The second preflight must pass all twelve installations +before any model trial starts; unsupported verifier profiles remain explicit. | Task | Coverage | Work budget (seconds) | | --- | --- | ---: | @@ -53,11 +65,11 @@ credentials through the process environment, never configuration or logs. Set ```bash python benchmarks/harbor/scripts/run_comparison.py \ - --plan benchmarks/harbor/configs/dev12-baseline-20261011/experiment.json \ + --plan benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v2.json \ --phase preflight python benchmarks/harbor/scripts/run_comparison.py \ - --plan benchmarks/harbor/configs/dev12-baseline-20261011/experiment.json \ + --plan benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v2.json \ --phase model ``` diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v2.json b/benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v2.json new file mode 100644 index 0000000..b718ea2 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v2.json @@ -0,0 +1,519 @@ +{ + "schema_version": 1, + "registered_at": "2026-10-11T15:30:40.768505+00:00", + "experiment": "dev12-main-baseline-20261011", + "selection_file": "selection.json", + "selection_sha256": "cd51acea6592c01ce9ec8cde34cd46a4965a4164950194f2de7e8ca54649bd29", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "arms": { + "main": { + "source_root": "jobs/dev12-baseline-main-source", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "prompt_file": "main-prompt.txt", + "prompt_sha256": "37c4d9d44640c0051dc5a20e5954a067b98e8c51be07a48f080aa059e725c55d", + "prompt_words": 385 + } + }, + "tasks": [ + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb", + "selection_rationale": "Native compilation and build tooling", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 12000, + "work_budget_seconds": 11820, + "verifier_timeout_seconds": 12000.0, + "category": "software-engineering", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/build-pov-ray:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "38380f64bc631916e264f80ce4d37feedbc783310001475b367d03989d114fdf", + "environment/Dockerfile": "44f2b5fc7311d8f336daaaca99dd4216b49355e67f50b494ebacf1b0f7c14e92", + "environment/deps/illum1.pov": "5a1f7f53ee0730c4923fcd21cd4a9c34719024d8557920441551ba7d96e75dc4", + "instruction.md": "56c8ecdd0084a07f8c97986c43af5c37ed03dd5aa994da6735cf59806162d1b2", + "solution/solve.sh": "3da9dbf589656512cd7a7004dc3752d3552a6af939cfe59eff79f91695fa42c7", + "task.toml": "b90a47ec26c1191d3c523cc094ea96b170e42fbfe176ef1f814c57d05390c097", + "tests/reference_illum1.png": "bb084f430bbc9d5b623a656920670dc06cee6d9132a9d550be4a1b99e41ab9e0", + "tests/test.sh": "928d1497ad21aef0c456294e3784f1fddf8493410f23584da89f9e02ec6bd134", + "tests/test_outputs.py": "64d6ce107503714ada346e5606832ee886be9950971cf81523e73d8ba28e8d37" + } + }, + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88", + "selection_rationale": "Legacy-language migration and behavioral compatibility", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "software-engineering", + "declared_difficulty": "easy", + "docker_image": "alexgshaw/cobol-modernization:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "c89318cf59efcc7040e3d946022e2616161b6733931a080cdeea7b1918e7ecc4", + "environment/Dockerfile": "8b175c83fc03ec7d16d6101516bc5100230b592f480e435279f4fe051c9280b2", + "environment/data/ACCOUNTS.DAT": "46f63eb29442ebc6716bcd8a0c36f4367910f2783f6b84a307ed9a61ccd1046e", + "environment/data/BOOKS.DAT": "db04da646207cd87f8e9c5b6f0b57d95953899f3eef22de1a81423f06ef4a9fd", + "environment/data/TRANSACTIONS.DAT": "1fa4cc5420ded94d585c563e8196b85166ebbae2bb8924b20014717fb2fb9ee3", + "environment/src/INPUT.DAT": "52fc984f0cf14af1495285d7efe02b9bae13159eea59255560ee938c4f78afee", + "environment/src/program.cbl": "27cb0e56f65f52fa75f346720b37a180b8e034ebe7d7622b979f195087bd6ebb", + "instruction.md": "a87074f1141b7345394cd88053d0debb0dc8f7de95308882c6538ee9a3b359da", + "solution/solve.sh": "7d4a0504e1686c1790bfbc074d4e598f4c631f54c2bc5f3d5fd786574b4810b9", + "task.toml": "4b620211dc3d6b1b989f0210c5ecd2f606974581c8bce27bd78659d48cf0cfbd", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "009180a7092583fc8143bb3ffc3957923aa0d61d0aac3a471f76eeb4aae54a88" + } + }, + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e", + "selection_rationale": "Native memory management debugging", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 1800, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "category": "debugging", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/custom-memory-heap-crash:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "a06ed23a2c467aa36d0d4a88e5c020b1076cc25b8d483517239658e9c596dde9", + "environment/Dockerfile": "6ee81e5a4fa2540443d7f27d577ba840728fc6aab9f301ab7919103673c78080", + "environment/patches/locale_classes.tcc.patch": "16951f4a4df6bf253886ac75f5a39fdbc328e415194493f63cb7589a364aa817", + "environment/patches/locale_init.cc.patch": "e97f133ceebd86f8881f972e0e45b0ec0a13a9bfa428d8032008707683d54292", + "environment/program/main.cpp": "b94538c76887cc2b2ab739b5ac4731cddd0faa8f4a9892a6265c062bbbeede14", + "environment/program/user.cpp": "92b203786fed1bf0b3aa1675c9cd2d82b2651170675064da8ffee73b98ef14f9", + "environment/program/user.h": "f43d155995a56c22b9cd1417daef7c3b5a976cdc662a98eef0feb438ae1eee77", + "instruction.md": "53c68ae14a3c3af9a97c40bfd9bdc30a5d98b37eaa2f69f763475745f70f108d", + "solution/solve.sh": "b0b1d10bea1cd5be14a82b35d522df616df1e67028f90d2a6bef90aaa98511a1", + "task.toml": "ed474be1983e212913aa8866ebb66216acbe36aef2339ac375d232aa46617481", + "tests/test.sh": "70a2b55f0c5757f6b123a216dfd202be4a646680bced28dddcd31fd8959a058f", + "tests/test_outputs.py": "4e1cb33f777fa7fb5e1536f9bea9eace6deccd515c67d4f87570adbe692914ba" + } + }, + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71", + "selection_rationale": "Security bug repair and behavior preservation", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "security", + "declared_difficulty": "hard", + "docker_image": "alexgshaw/fix-code-vulnerability:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "87f28dbad183c3a634887696d328ca68a8a0858f132bc48cb7341c22267d38f7", + "environment/Dockerfile": "0917470986224df4daf4854a71302ae81c73ba5dd6e25be1232d29b758a5b3ab", + "instruction.md": "89b1abdf0af19399f720233b5cdf6b2e8e0ff7b6e3fe3e2a3af0d4d298a8641a", + "solution/solve.sh": "7a0131b369d83ae8546ecc3f9b4e23d38240f1bd1f0941e4430fa85b5182ee03", + "task.toml": "9aa2bd14ce9408289d199d0608bc3fa65b8b5e23374c9641020de62197dbf1e2", + "tests/test.sh": "f1611da693cb4cfc5e8482698c6a1858a5c67220ae9db89b50aed66584e9b17f", + "tests/test_outputs.py": "c006a5de95b69bd78055aa26cc055734dbd50718ede25d331f767bfee4ef7f26" + } + }, + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097", + "selection_rationale": "Service configuration and observable request behavior", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "system-administration", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/nginx-request-logging:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d79997586d1c14ed8dbfea050e3580b25500e04c8345acfd3f294152433c795a", + "environment/Dockerfile": "43003466d89ab83484a3fa618e35b58cdb74a002b1e8716ef30f12b3ac8302d8", + "instruction.md": "26a7ac98aced6107f147206790ada77bbf8ed3c25fb360cf323d5fc6889edf99", + "solution/solve.sh": "2454bbbf7c201a9a778165f286373ba2cb991a0eb1d026a8005d57ae71250dd9", + "task.toml": "b7a78d00be13656f75069197100b7b8c4fd268decf6f78929c853b3222a2729d", + "tests/test.sh": "27761195f53cbfe5941fe39b573631f6c926869e64b7c43220377e1e116528e7", + "tests/test_outputs.py": "045cc716c14efde3b0dcff5fc7c85ec5d18bfc6ce66f8b40a418fa2a3a4acda0" + } + }, + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa", + "selection_rationale": "Cross-branch repository operations", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "system-administration", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/git-multibranch:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "3b9473429381d9eacabab498cb4a8915aa2bca24fa857ef862e5991d3f81f113", + "environment/Dockerfile": "88d9a9f4fef1ae7fddf632799dedf5a2fb6489460a93b610c33dd8b122982821", + "environment/default.conf": "a3a4c5d0ac9cfe79b0510e4225c80fa3988c27fbac13622505886bf66449e04a", + "instruction.md": "54014a38d7526d65faeb6c11d9b7ed3d4a5cb5d973d68d3da3b68b8c0289e125", + "solution/solve.sh": "300ca96b79db8d77f14742109aaf76048aafc4644ced750b1bc5c2761e8d0175", + "task.toml": "4b9981604578a31205257cca36a28d0c89c9d8d1058541330acd11d3f8d65f82", + "tests/test.sh": "ddbde3eea6ffa8fda105cbd86597a53bd409f41e13223900aa27e77a84b6b155", + "tests/test_outputs.py": "f03c6867663b8e8fb7899df51ee4df26411fab62ed491753964b801e591d16c5" + } + }, + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4", + "selection_rationale": "Database recovery and data integrity", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "file-operations", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/db-wal-recovery:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "600762031a1a02ecc3b46d70f7683f3ff930b47c8a3d0e27cc31b146158c3126", + "environment/Dockerfile": "63a260531c62f4ae72060db17f295c0aa03cfc6321f8162c324b7aa6b3f81302", + "environment/main.db": "68cb288eb29e76d814b4e7ee6887b37c415675e644f88173bc191348e290d119", + "environment/main.db-wal.encrypted": "68753bdce26196a53194260d001544cc6bb8f9f38e2f2f6826d09798e31e6e26", + "instruction.md": "5a20375162921b8911c67035d1905df7fd335d1bed03044271e608b2c3c93b26", + "solution/solve.sh": "ed8530c94103bf19783a1d41bc63fe4db478cdd9629f22fee25c45a45699f5e8", + "task.toml": "1cdc2c615e259bc10265efbe71b518332070c1c002e226566399e9788f18d0aa", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "88dc5c2a3abba75bca0e0f9136223ae5bb585d5984b8d9eba51a7829c1d2ba49" + } + }, + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49", + "selection_rationale": "Data integration and cross-input consistency", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "data-processing", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/multi-source-data-merger:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9bf37162a4c1b88ed93627830bb8512ebae154325d7cb069c883144a0e4c69d6", + "environment/Dockerfile": "c95a40d3d35b473dc15ae1e8784ba7fb1725805c0583fc574dd2764325d75e4f", + "environment/data/source_a/users.json": "75e51e2d3ab0bdba2269d7cf3757576f84f3ee2c13407d2db94289a5958a1778", + "environment/data/source_b/users.csv": "8e87d16dc47d166edd8e56165955bad265b630d53d02762367c816affb5fa760", + "environment/data/source_c/users.parquet": "23d1ea51c4170bdba53febb885a0f495e28de525a52667347b2932136b9a72fe", + "instruction.md": "d1d28bb6e37f7a255a1b7f78aae8591f35c0494f33e4913392424ad954aab944", + "solution/solve.sh": "1b3d0d50e31930d042495d48f4c8e95f1a9c1a979c803b7f7cec651322537e1e", + "task.toml": "ee3964c0ee040f0a0cb121cefefbc15443d127684acb003fa5b61dd42f31c694", + "tests/test.sh": "d5a00e64413c0e1ae452f38a1aa1a47c3ed7b4416bbfccae1a4c053025a336d1", + "tests/test_outputs.py": "a070888930cf829aeca039771260b1386bda990ea805f313468d381ba3608ca5" + } + }, + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4", + "selection_rationale": "Database query performance and correctness", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 1800.0, + "category": "data-science", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/query-optimize:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "a82c53e333a0ac0b0b410b5e26cb8e7305dece31db84d340248ffb13d5a45160", + "environment/Dockerfile": "064818771f200e0517d22068f87229a7a95d0a82b0778c84ca7203bf1926a0b8", + "environment/task-deps/my-sql-query.sql": "43382792f0f438625d643e34abe144febd5669518f4ab1ab550a15c6d9314a21", + "instruction.md": "473e12d320d2732f679fb631c2a4dab4a01deea6775686718ae0c417b8a39447", + "solution/solve.sh": "0d95f3cedb287ff9d47c796cbaf812c300d1222c10fa80c49e8acb0e81eb6978", + "task.toml": "6e88f2a86df699594bcb1bf81df4ed694505f79c9420f671335b02524bbe3868", + "tests/golden.sql": "4da1885d56b3ffa7b7b22b422996d018795611f6963499d551582881b74b84fd", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "2068ae80c57af8d52a7ae510f0666732c057806e556b480970f36f14bae683f6" + } + }, + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de", + "selection_rationale": "Numerical sampling and statistical invariants", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "scientific-computing", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/adaptive-rejection-sampler:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "95408217b9c484ffc18a02a91b7706323b6902919c72b24866a32e27a23543e2", + "environment/Dockerfile": "e644a3343e339303230f1441b7383f076d21c69a5a7aa6430b4f8a9d27f15c8b", + "environment/protected.tar.gz.enc": "7fc9bf1347bb97da8dede04b2c58638f225070913e222d76627068aebd1b1df2", + "instruction.md": "e6d28c28acd76bef61817427c74c2a29e3e8d75f535cbd708afcbfd02c54013f", + "solution/solve.sh": "fe443a8082b5183ba1623311ba0ee958888aefa633eff93766b9ae14e9bdd84d", + "task.toml": "6ed7b37f74beb46e9d47b0f5fad53025667c9548baff4e96d467b56d429fab1a", + "tests/test.sh": "38b43560d173cc2b952c3a3e17b8a480216d84e33450515e047bcb0d806b1e0a", + "tests/test_outputs.py": "547dc6e107f034f41703722aeceb6d0236e3fb69116fc3f2fbaba11884de352f" + } + }, + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53", + "selection_rationale": "Constrained algorithmic planning", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 1200, + "work_budget_seconds": 1020, + "verifier_timeout_seconds": 1200.0, + "category": "personal-assistant", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/constraints-scheduling:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d54a690970149e1216bd4fc066b45069c7948c13e3b1ef2a213a74342805f209", + "environment/Dockerfile": "7d9aba072a4a907229728b9739c8cf821fc2c98450babd64452682caf3759c84", + "environment/inputs/alice_calendar.ics": "5a5d45804bffd339c2479d49e76fe817b38244d14d389afe1de583443808ecbc", + "environment/inputs/bob_calendar.ics": "c97e85dbcdd756758f8a80f4db5fd5c1dcacee79298e4d2262f46de5ff680b50", + "environment/inputs/carol_calendar.ics": "32a0f8687637df0a12b8e5c56d49c4192bc3e790856943632d70f237c7c1363b", + "instruction.md": "342def78177f9bc41bea4d11af679bc385a9dda2ac8a1e425be384a5b7f937b6", + "solution/solve.sh": "802d81ec064fad3050bbade95a2f847ac24e0b7f80fa69b0bcee0347dc92fe77", + "task.toml": "7b9842de16d74e8faf8c7c0aa03b0b81145941cb12dd322d81ae23fc48607250", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "1422b345ba3aacdfc4184e962811b6379988435896c50d8cf62903e70990c5d1" + } + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79", + "selection_rationale": "Model packaging and command-line contracts", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "model-training", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/pytorch-model-cli:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "35a093fa5f1592737784ba718c2e3e9ae17ff8b5e53e003fae560ddce091039c", + "environment/Dockerfile": "8fd5c0f21a6914cd440d10c84c9db06fec606f20dc3e0363b17a3b8c220dc4d9", + "environment/task-deps/cJSON.c": "0e7995261640c96db16ed2c91a5d981efa917cf5e72d0f97ec45a1179ae3ad73", + "environment/task-deps/cJSON.h": "0909160541bcc510e0b2d8c7a9cc051fa1459a0f13be4d8e56346a15491f2f2f", + "environment/task-deps/image.png": "f4d5dfefe463939996541404340e7099d5957b56e1df3760d13b5d2de754db53", + "environment/task-deps/lodepng.cpp": "67873c5f6272f779a143a9c81070e8783327c5552d0b9de2aaee2cb779308917", + "environment/task-deps/lodepng.h": "b735b7e0357319973e9488d520d7f270d734a6e9e1bf924fb1d5b8d26d7e97ce", + "environment/task-deps/model.pth": "a32103eef09936e586b1a7dc7e64d279500a38ff30fdd6df2167e28eb954eee6", + "environment/task-deps/model.py": "e4d3a5c728449656d0efde7871ee227619348352769795ed1be1a8b7f835fddc", + "instruction.md": "ce36ee1f6c589f81ee6a6be4967e25c76a2ed697006928023f42dfb9f28a6750", + "solution/solve.sh": "3b34e561c9ef0a53c9948655847846eb4811aaac67b990480aa6aac1b04bfe66", + "task.toml": "fc1c018efbd5d0537ac0741d53d1ebad3886faa51d14cbde8ec953856411bc0d", + "tests/test.sh": "43fb67e537c7adc2df11c2367f47c876cb5d16a7c1ae16d0339c3a3364bb2b06", + "tests/test_outputs.py": "370702c57a4e0c85f70ee37a5c357ae3a5df8480a275bb6263b0376f2f6c093b" + } + } + ], + "total_model_runs": 12, + "max_concurrent_trials": 2, + "whole_trial_retries": 0, + "preflight": [ + { + "arm": "main", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/install-only.json", + "config_sha256": "b8fc1d1d825cc514c345026bdfce4cd867443125aac0de1d0d6e02b6c3e7c30a", + "job_name": "dev12-main-install-only-v2-20261011" + } + ], + "queue": [ + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/cobol-modernization", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/cobol-modernization.json", + "job_name": "dev12-cobol-modernization-baseline-20261011", + "config_sha256": "2680126c2f162d58948cd30565806bf6cb1f026d561c87b942ba4735173ea6a2" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/fix-code-vulnerability", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/fix-code-vulnerability.json", + "job_name": "dev12-fix-code-vulnerability-baseline-20261011", + "config_sha256": "4b2de43b547ddd92d748f7658c44b98b4fef25efd406710ace6d1713b9c789a7" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/nginx-request-logging", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/nginx-request-logging.json", + "job_name": "dev12-nginx-request-logging-baseline-20261011", + "config_sha256": "4a79d4fc4431acd942548f36188c1820ac5406fc6a1385cf505daf02aa190da3" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/git-multibranch", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/git-multibranch.json", + "job_name": "dev12-git-multibranch-baseline-20261011", + "config_sha256": "ae9c6d5830c9dcb8aab8b1356441b47731957e2ad5e5747327b7caa1a6c037a1" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/db-wal-recovery", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/db-wal-recovery.json", + "job_name": "dev12-db-wal-recovery-baseline-20261011", + "config_sha256": "475b7df31549e51f04f4f9171332da5a8ba03f7fbec2b3788fbf897340c0c91b" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/multi-source-data-merger", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/multi-source-data-merger.json", + "job_name": "dev12-multi-source-data-merger-baseline-20261011", + "config_sha256": "5eac4294149a11f037f732a32e1a8874613d8a28884843955481a9e8ef10b7bd" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/query-optimize", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/query-optimize.json", + "job_name": "dev12-query-optimize-baseline-20261011", + "config_sha256": "29ad0478282cd6fb86a1b209a93ffe33eee724c321916732db001b7beef8ea7e" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/adaptive-rejection-sampler", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/adaptive-rejection-sampler.json", + "job_name": "dev12-adaptive-rejection-sampler-baseline-20261011", + "config_sha256": "e0fec8e3f1f8e077a5384058ea5b6e4733973a7475b02556f90cfc0f4ba96f6d" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/pytorch-model-cli", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/pytorch-model-cli.json", + "job_name": "dev12-pytorch-model-cli-baseline-20261011", + "config_sha256": "bc915a7de1286954473271e2232f9c83af983c23b885d099316b33dc3c1d445d" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/constraints-scheduling", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/constraints-scheduling.json", + "job_name": "dev12-constraints-scheduling-baseline-20261011", + "config_sha256": "33813c2fd5b45efd3b7bf8a765fcb1f74c702821a3d4f473e4b479200433734e" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/custom-memory-heap-crash", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/custom-memory-heap-crash.json", + "job_name": "dev12-custom-memory-heap-crash-baseline-20261011", + "config_sha256": "4678ab6961d82a2d597911bd8d4a53040a4be526c1e97afb27c614ecedb1c05f" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/build-pov-ray", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/build-pov-ray.json", + "job_name": "dev12-build-pov-ray-baseline-20261011", + "config_sha256": "42e1bb7d2a1a6d5cf9f059a121140f9135d8b4632482a00d767a9aba2e1ac2e7" + } + ], + "log_dir": "jobs/dev12-baseline-20261011-v2-orchestration", + "budget_policy": "Native task agent timeout minus 180 seconds for runtime work; setup separately allows 1800 seconds. Native verifier limits remain unchanged. Build POV-Ray retains its unusually long 12000-second native limit (11820-second model work).", + "execution_policy": "One run per selected task. Run all twelve installation preflights before models. Queue is fixed by ascending native work budget, stable within ties; run consecutive pairs with at most two concurrent model trials.", + "failure_policy": "Retain every selected trial including setup failures, verifier errors and scored failures; no outcome-based substitutions or model reruns. Only verifier timeouts may retry, at most three total attempts on the same output with the native per-attempt limit.", + "preflight_policy": "All selected verifier dependency profiles are not_configured. Agent installation success does not certify verifier dependencies. Registered profiles, if present, must pass before model work.", + "information_policy": "Selection used task names before downloading. Only public metadata/instructions may be inspected before every agent output freezes; task files are hashed as bytes without inspecting hidden test or solution contents. Audit external references afterwards.", + "scope": "Development baseline for twelve new tasks, not a representative full-dataset score or a pass-rate improvement experiment. Existing 22 task records remain separate; remaining 55 tasks are not opened.", + "protocol_amendment": { + "supersedes": "experiment.json", + "supersedes_sha256": "88b2c2391b6c00cbdb03408d8af39d32c0d79266ea2b02913a3537224213c8ab", + "evidence_file": "preflight-v1-evidence.json", + "evidence_sha256": "c723c858571a512e968db16babed482a2874f270561ae3ae8c426a2ad74bd73f", + "reason": "The first install-only job encountered pre-model infrastructure failures recorded in the evidence file. Preserve every failure, cache the exact original images, and repeat all twelve installation checks under a new registration before any model work.", + "changed_fields": [ + "registered_at", + "log_dir", + "preflight[0].job_name", + "protocol_amendment" + ], + "unchanged": "Task selection and revisions, runtime and prompt, model, all task configurations and budgets, model queue and job names, concurrency, retry and information policies.", + "model_runs_started_at_amendment": 0 + } +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/preflight-v1-evidence.json b/benchmarks/harbor/configs/dev12-baseline-20261011/preflight-v1-evidence.json new file mode 100644 index 0000000..167fab3 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/preflight-v1-evidence.json @@ -0,0 +1,416 @@ +{ + "recorded_at": "2026-10-11T15:30:40.768505+00:00", + "original_plan_sha256": "88b2c2391b6c00cbdb03408d8af39d32c0d79266ea2b02913a3537224213c8ab", + "preflight_state_path": "jobs/dev12-baseline-20261011-orchestration/dev12-main-install-only-20261011.json", + "preflight_state_sha256": "dc794453b094c36d3cd28679a006d6178bb61f90d28f875b2957c8fefe0aa2ff", + "preflight_exit_code": 1, + "model_runs_started": 0, + "trials": [ + { + "task": "terminal-bench/adaptive-rejection-sampler", + "trial": "adaptive-rejection-sampler__RF8AnGv", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/adaptive-rejection-sampler__RF8AnGv/result.json", + "result_sha256": "2f2c44870848bfa6155c7d2f31f1c1b2a734ca783e050cd344ecb22262e8125b", + "started_at": "2026-10-11T14:52:09.897066Z", + "finished_at": "2026-10-11T15:29:53.170833Z", + "exception_type": "AgentSetupTimeoutError", + "bootstrap": { + "status": "failed", + "stage": "system_dependencies", + "error_type": "CancelledError", + "cache_used": false, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "apt_requested_packages": [] + }, + "model_started": false, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/adaptive-rejection-sampler__RF8AnGv/agent/bootstrap.json", + "bootstrap_sha256": "82878d7f8a339f6a19b1130e8160cbd8e23b500246887197f67728670a0e05a8" + }, + { + "task": "terminal-bench/build-pov-ray", + "trial": "build-pov-ray__JBS6iy8", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/build-pov-ray__JBS6iy8/result.json", + "result_sha256": "c878116d4f2746c3269ceefba34b9f0d1c0d9672d3010adb43ff904767391c84", + "started_at": "2026-10-11T14:37:44.785883Z", + "finished_at": "2026-10-11T14:39:13.586321Z", + "exception_type": null, + "bootstrap": { + "status": "ready", + "stage": "complete", + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "cache_used": true, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "apt_requested_packages": [ + [ + "curl", + "bash", + "git", + "python3", + "ca-certificates" + ] + ] + }, + "model_started": false, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/build-pov-ray__JBS6iy8/agent/bootstrap.json", + "bootstrap_sha256": "40256d355dfe43492278a1a4a22b8a6093657509086254ddd27494162c85b0b4" + }, + { + "task": "terminal-bench/cobol-modernization", + "trial": "cobol-modernization__vX5nBDX", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/cobol-modernization__vX5nBDX/result.json", + "result_sha256": "65726b6e867002ac7e0306d3e1763895076389d164a9b6bd611cab89fc28f021", + "started_at": "2026-10-11T14:37:44.880661Z", + "finished_at": "2026-10-11T14:39:09.924816Z", + "exception_type": null, + "bootstrap": { + "status": "ready", + "stage": "complete", + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "cache_used": true, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "apt_requested_packages": [ + [ + "curl", + "bash", + "git", + "python3", + "ca-certificates" + ] + ] + }, + "model_started": false, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/cobol-modernization__vX5nBDX/agent/bootstrap.json", + "bootstrap_sha256": "1812b1d7452b4ec930ab65be85cd8ba7f2e96342153c5e9e7d0dc948f342502f" + }, + { + "task": "terminal-bench/constraints-scheduling", + "trial": "constraints-scheduling__3qnvwvS", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/constraints-scheduling__3qnvwvS/result.json", + "result_sha256": "f01f3f862c70c246f4f0b1ab0617d7fbd02ffed99df63eeed39b8484485734b7", + "started_at": "2026-10-11T15:02:12.907941Z", + "finished_at": "2026-10-11T15:12:14.134571Z", + "exception_type": "EnvironmentStartTimeoutError", + "bootstrap": null, + "model_started": false + }, + { + "task": "terminal-bench/custom-memory-heap-crash", + "trial": "custom-memory-heap-crash__feSC86P", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/custom-memory-heap-crash__feSC86P/result.json", + "result_sha256": "7d49cf4504bca281dd3115918e3efedea9aea9f4b0bbac63f6a1b33742446b84", + "started_at": "2026-10-11T14:39:12.622251Z", + "finished_at": "2026-10-11T14:41:58.270096Z", + "exception_type": null, + "bootstrap": { + "status": "ready", + "stage": "complete", + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "cache_used": true, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "apt_requested_packages": [ + [ + "curl", + "bash", + "git", + "python3", + "ca-certificates" + ] + ] + }, + "model_started": false, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/custom-memory-heap-crash__feSC86P/agent/bootstrap.json", + "bootstrap_sha256": "713cf8bea3c485e3e227a8737b6f21b5e1c41cb5b990897d3f5fa7a5445dfae5" + }, + { + "task": "terminal-bench/db-wal-recovery", + "trial": "db-wal-recovery__Nuxi3FJ", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/db-wal-recovery__Nuxi3FJ/result.json", + "result_sha256": "dfc37eb35223f1ff09cf2a6e6dc47248e4caafb8d5d164461a82404604d4f6ee", + "started_at": "2026-10-11T14:42:03.834053Z", + "finished_at": "2026-10-11T14:52:04.644330Z", + "exception_type": "EnvironmentStartTimeoutError", + "bootstrap": null, + "model_started": false + }, + { + "task": "terminal-bench/fix-code-vulnerability", + "trial": "fix-code-vulnerability__zXDnZib", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/fix-code-vulnerability__zXDnZib/result.json", + "result_sha256": "70179ecad359af42a76481d4099f100e9860605410b724c732e731a8f567d967", + "started_at": "2026-10-11T14:39:14.249670Z", + "finished_at": "2026-10-11T14:40:30.306384Z", + "exception_type": null, + "bootstrap": { + "status": "ready", + "stage": "complete", + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "cache_used": false, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "apt_requested_packages": [ + [ + "curl", + "bash", + "git", + "python3", + "ca-certificates" + ] + ] + }, + "model_started": false, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/fix-code-vulnerability__zXDnZib/agent/bootstrap.json", + "bootstrap_sha256": "0ebcaf72b1671da138d80d344700a33c9f8cf6364e54193732d28806218c24fc" + }, + { + "task": "terminal-bench/git-multibranch", + "trial": "git-multibranch__vmNqLCo", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/git-multibranch__vmNqLCo/result.json", + "result_sha256": "4390e038e6370493d9fc1b2269f00ad0c40d9da987fed3be056e7ae38241b89a", + "started_at": "2026-10-11T14:42:00.835897Z", + "finished_at": "2026-10-11T14:42:08.522946Z", + "exception_type": "RuntimeError", + "bootstrap": null, + "model_started": false + }, + { + "task": "terminal-bench/multi-source-data-merger", + "trial": "multi-source-data-merger__sSrB3BY", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/multi-source-data-merger__sSrB3BY/result.json", + "result_sha256": "29282679b5ffb1a5f8e6b39b397f18718856bb06224a1735f8003aa0ac0f6148", + "started_at": "2026-10-11T14:42:08.843804Z", + "finished_at": "2026-10-11T14:52:09.624139Z", + "exception_type": "EnvironmentStartTimeoutError", + "bootstrap": null, + "model_started": false + }, + { + "task": "terminal-bench/nginx-request-logging", + "trial": "nginx-request-logging__TtmcRmS", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/nginx-request-logging__TtmcRmS/result.json", + "result_sha256": "21be2091123469444698c92f126a6876ea78a32b3b75f05bcc8e43a33659b383", + "started_at": "2026-10-11T14:40:32.151294Z", + "finished_at": "2026-10-11T14:42:01.242999Z", + "exception_type": null, + "bootstrap": { + "status": "ready", + "stage": "complete", + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "cache_used": true, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "apt_requested_packages": [ + [ + "curl", + "bash", + "git", + "python3", + "ca-certificates" + ] + ] + }, + "model_started": false, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/nginx-request-logging__TtmcRmS/agent/bootstrap.json", + "bootstrap_sha256": "bd617a3903bf50c818febe3abf77cc268952deb3cd9f0c6db99bc0cbf8146a94" + }, + { + "task": "terminal-bench/pytorch-model-cli", + "trial": "pytorch-model-cli__LSDgEmN", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/pytorch-model-cli__LSDgEmN/result.json", + "result_sha256": "396ffe0341f26fb3b619bc6f8a8421e3eafc343480c7c10b89967feb5c68470b", + "started_at": "2026-10-11T15:12:16.564097Z", + "finished_at": "2026-10-11T15:22:16.649591Z", + "exception_type": "EnvironmentStartTimeoutError", + "bootstrap": null, + "model_started": false + }, + { + "task": "terminal-bench/query-optimize", + "trial": "query-optimize__3swyr89", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/query-optimize__3swyr89/result.json", + "result_sha256": "b0098f4041e7f553f7e036d3e52bfcb34c196bdcb76c09ec54dc60145b2eb1ff", + "started_at": "2026-10-11T14:52:08.733745Z", + "finished_at": "2026-10-11T15:02:08.667707Z", + "exception_type": "EnvironmentStartTimeoutError", + "bootstrap": null, + "model_started": false + } + ], + "original_images_cached_before_second_preflight": [ + { + "task": "terminal-bench/build-pov-ray", + "image": "alexgshaw/build-pov-ray:20251031", + "image_id": "sha256:874baf49341e2853a15a9184e308c7eb632e97f434359fcff2891c6e5ca1b39b", + "repo_digests": [ + "alexgshaw/build-pov-ray@sha256:874baf49341e2853a15a9184e308c7eb632e97f434359fcff2891c6e5ca1b39b" + ], + "size_bytes": 163066497 + }, + { + "task": "terminal-bench/cobol-modernization", + "image": "alexgshaw/cobol-modernization:20251031", + "image_id": "sha256:593ab9df3d83f771e927888b2d9436b380cdf7ec4d0af2a9f0802d541edebba0", + "repo_digests": [ + "alexgshaw/cobol-modernization@sha256:593ab9df3d83f771e927888b2d9436b380cdf7ec4d0af2a9f0802d541edebba0" + ], + "size_bytes": 162020401 + }, + { + "task": "terminal-bench/custom-memory-heap-crash", + "image": "alexgshaw/custom-memory-heap-crash:20251031", + "image_id": "sha256:1895d7d6240d5fb2ec774d6bfdd1057d170f8d199c15c582e4585d3c63802557", + "repo_digests": [ + "alexgshaw/custom-memory-heap-crash@sha256:1895d7d6240d5fb2ec774d6bfdd1057d170f8d199c15c582e4585d3c63802557" + ], + "size_bytes": 1267999268 + }, + { + "task": "terminal-bench/fix-code-vulnerability", + "image": "alexgshaw/fix-code-vulnerability:20251031", + "image_id": "sha256:cac325252991f823713b2d0441502972901dd782bd67f66c03d9b1e410dac5c0", + "repo_digests": [ + "alexgshaw/fix-code-vulnerability@sha256:cac325252991f823713b2d0441502972901dd782bd67f66c03d9b1e410dac5c0" + ], + "size_bytes": 190696453 + }, + { + "task": "terminal-bench/nginx-request-logging", + "image": "alexgshaw/nginx-request-logging:20251031", + "image_id": "sha256:693e64431a0dbf45952e8b91a298d54fd60247e501c6623fb8fa3ebc07fc3d3d", + "repo_digests": [ + "alexgshaw/nginx-request-logging@sha256:693e64431a0dbf45952e8b91a298d54fd60247e501c6623fb8fa3ebc07fc3d3d" + ], + "size_bytes": 67285778 + }, + { + "task": "terminal-bench/git-multibranch", + "image": "alexgshaw/git-multibranch:20251031", + "image_id": "sha256:5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5", + "repo_digests": [ + "alexgshaw/git-multibranch@sha256:5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5" + ], + "size_bytes": 222134454 + }, + { + "task": "terminal-bench/db-wal-recovery", + "image": "alexgshaw/db-wal-recovery:20251031", + "image_id": "sha256:0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e", + "repo_digests": [ + "alexgshaw/db-wal-recovery@sha256:0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e" + ], + "size_bytes": 175276145 + }, + { + "task": "terminal-bench/multi-source-data-merger", + "image": "alexgshaw/multi-source-data-merger:20251031", + "image_id": "sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e", + "repo_digests": [ + "alexgshaw/multi-source-data-merger@sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e" + ], + "size_bytes": 220880128 + }, + { + "task": "terminal-bench/query-optimize", + "image": "alexgshaw/query-optimize:20251031", + "image_id": "sha256:28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14", + "repo_digests": [ + "alexgshaw/query-optimize@sha256:28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14" + ], + "size_bytes": 120494963 + }, + { + "task": "terminal-bench/adaptive-rejection-sampler", + "image": "alexgshaw/adaptive-rejection-sampler:20251031", + "image_id": "sha256:985bed39e8448bc95f138b5f766b59003a95c4be171c96af8e76448338f3dd8f", + "repo_digests": [ + "alexgshaw/adaptive-rejection-sampler@sha256:985bed39e8448bc95f138b5f766b59003a95c4be171c96af8e76448338f3dd8f" + ], + "size_bytes": 29731756 + }, + { + "task": "terminal-bench/constraints-scheduling", + "image": "alexgshaw/constraints-scheduling:20251031", + "image_id": "sha256:567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0", + "repo_digests": [ + "alexgshaw/constraints-scheduling@sha256:567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0" + ], + "size_bytes": 49852952 + }, + { + "task": "terminal-bench/pytorch-model-cli", + "image": "alexgshaw/pytorch-model-cli:20251031", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "repo_digests": [ + "alexgshaw/pytorch-model-cli@sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67" + ], + "size_bytes": 44349611 + } + ], + "image_cache_utility": { + "path": "benchmarks/harbor/scripts/cache_image.py", + "source_commit": "d7dbcec", + "source_sha256": "639709c03a7b0c3eb6aa78b4e87e6184b49b4b102853a31e33dd832d759180c0", + "unit_and_harbor_tests_passed": 113, + "real_docker_import_digest_check": "passed" + }, + "imported_manifest_locks": [ + { + "image": "alexgshaw/constraints-scheduling:20251031", + "manifest_digest": "sha256:567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0", + "config_digest": "sha256:65694556a87deb6f2c0f643841d4d4ff3fabd1e794999978694d891ccb24e703" + }, + { + "image": "alexgshaw/db-wal-recovery:20251031", + "manifest_digest": "sha256:0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e", + "config_digest": "sha256:0ff6d2977f94533d8a7a0cdc5a16efd42cb3ccfeb2f5684cdc8db0836b43b9e3" + }, + { + "image": "alexgshaw/git-multibranch:20251031", + "manifest_digest": "sha256:5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5", + "config_digest": "sha256:ec6eeb02cecf35faf5badd75646ab0dd2e02cab484638b413ccd4606a2e2ef7d" + }, + { + "image": "alexgshaw/multi-source-data-merger:20251031", + "manifest_digest": "sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e", + "config_digest": "sha256:5096d3409e5c692afd62a136f90a2e52dd87653aa6637a3e1ca31ad30fab1096" + }, + { + "image": "alexgshaw/pytorch-model-cli:20251031", + "manifest_digest": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "config_digest": "sha256:2d19236650cc866ba441762e3d89edfa452756e0e4833cfdb984257d50b446e7" + }, + { + "image": "alexgshaw/query-optimize:20251031", + "manifest_digest": "sha256:28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14", + "config_digest": "sha256:5654e241057913a0f1a234583c62e12ed7bfab560d9c3f0dc5da985d9f3514eb" + } + ], + "first_preflight_summary": { + "setup_passed": 5, + "image_pull_or_environment_start_errors": 6, + "agent_setup_timeouts": 1, + "model_runs_started": 0 + }, + "cleanup": { + "at": "2026-10-11T15:30:36.144215+00:00", + "preflight_job": "dev12-main-install-only-20261011", + "stopped_orphan_compose_clients": [], + "removed_finished_preflight_containers": [], + "model_executions_in_cleaned_trials": 0 + } +} From 491ee92c460772c9cd26204e31e4198c7b9e4948 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Sun, 11 Oct 2026 08:50:15 -0700 Subject: [PATCH 4/5] test(benchmarks): verify reusable installation evidence before baseline --- .../configs/dev12-baseline-20261011/README.md | 37 +- .../experiment-v3.json | 625 ++++++++++++++++++ .../preflight-v2-evidence.json | 206 ++++++ benchmarks/harbor/scripts/run_comparison.py | 74 ++- benchmarks/harbor/tests/test_comparison.py | 106 +++ 5 files changed, 1033 insertions(+), 15 deletions(-) create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v3.json create mode 100644 benchmarks/harbor/configs/dev12-baseline-20261011/preflight-v2-evidence.json diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/README.md b/benchmarks/harbor/configs/dev12-baseline-20261011/README.md index d549a6e..c1ce6c3 100644 --- a/benchmarks/harbor/configs/dev12-baseline-20261011/README.md +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/README.md @@ -15,18 +15,29 @@ is included. The fixed dataset revision is [Selection](selection.json) records the existing 22 tasks, selected revisions and rationales. The [original registered plan](experiment.json) records runtime, prompt, configuration and task-file hashes, work limits, preflight and queue -order. The active [pre-model revision](experiment-v2.json) repeats installation -preflight under a new job name and registration directory after infrastructure -failures; all model configurations, tasks, runtime, budgets and queue entries -remain identical. +order. The active [pre-model revision](experiment-v3.json) accepts twelve +successful native installation trials with hash-bound provenance. All model +configurations, tasks, runtime, budgets and queue entries remain identical. The [first preflight evidence](preflight-v1-evidence.json) retains five successful installations, six image-pull/environment-start errors, and one agent setup timeout while `apt-get update` was still downloading package indexes. No models or official verifiers ran. All twelve original images are now cached, including six downloaded with the verified image-cache utility. Their original registry -digests are recorded. The second preflight must pass all twelve installations -before any model trial starts; unsupported verifier profiles remain explicit. +digests are recorded. + +The [second preflight](experiment-v2.json) completed eleven successful +installations; POV-Ray's uv download ended with a partial-transfer error. +[Its evidence](preflight-v2-evidence.json) retains every result. The final plan +uses these eleven successes and POV-Ray's earlier successful installation, +without replaying successful installations because of another trial's network +failure. Before model work, the orchestrator requires exactly one accepted +installation-only result for every queued task and source arm. It verifies the +task revision, source snapshot, installed wheel, bootstrap status, input mode, +and recorded artifact hashes. A scored trial, failed installation, pending +registered verifier profile, incomplete set, or changed evidence is rejected. +Unsupported verifier profiles remain explicit. Both failed jobs and all eight +failed preflight trials remain in the evidence; none is a scored agent failure. | Task | Coverage | Work budget (seconds) | | --- | --- | ---: | @@ -65,20 +76,24 @@ credentials through the process environment, never configuration or logs. Set ```bash python benchmarks/harbor/scripts/run_comparison.py \ - --plan benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v2.json \ + --plan benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v3.json \ --phase preflight python benchmarks/harbor/scripts/run_comparison.py \ - --plan benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v2.json \ + --plan benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v3.json \ --phase model ``` The reusable orchestrator calls only the standard `uv run --project benchmarks/harbor python -m harbor_adapter run --config ...` workflow from the frozen repository root. It creates an immutable registration, -requires all twelve install-only results before model work, and refuses to retry -a job already recorded as started. Source snapshots and wheel manifests remain -in the normal Harbor job-input directories. +requires all twelve successful install-only records before model work, and +refuses to retry a job already recorded as started. With the final plan, +`--phase preflight` validates the recorded successful installations without +launching another installation job. Source snapshots and wheel manifests remain +in the normal Harbor job-input directories. A full model trial still installs +the agent in its own fresh container; preflight does not eliminate later +network failures. All twelve selected verifier dependency profiles are explicitly `not_configured`. Agent installation preflight does **not** certify verifier diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v3.json b/benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v3.json new file mode 100644 index 0000000..60f7489 --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v3.json @@ -0,0 +1,625 @@ +{ + "schema_version": 1, + "registered_at": "2026-10-11T15:48:27.483536+00:00", + "experiment": "dev12-main-baseline-20261011", + "selection_file": "selection.json", + "selection_sha256": "cd51acea6592c01ce9ec8cde34cd46a4965a4164950194f2de7e8ca54649bd29", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "arms": { + "main": { + "source_root": "jobs/dev12-baseline-main-source", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "prompt_file": "main-prompt.txt", + "prompt_sha256": "37c4d9d44640c0051dc5a20e5954a067b98e8c51be07a48f080aa059e725c55d", + "prompt_words": 385 + } + }, + "tasks": [ + { + "name": "terminal-bench/build-pov-ray", + "ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb", + "selection_rationale": "Native compilation and build tooling", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 12000, + "work_budget_seconds": 11820, + "verifier_timeout_seconds": 12000.0, + "category": "software-engineering", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/build-pov-ray:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "38380f64bc631916e264f80ce4d37feedbc783310001475b367d03989d114fdf", + "environment/Dockerfile": "44f2b5fc7311d8f336daaaca99dd4216b49355e67f50b494ebacf1b0f7c14e92", + "environment/deps/illum1.pov": "5a1f7f53ee0730c4923fcd21cd4a9c34719024d8557920441551ba7d96e75dc4", + "instruction.md": "56c8ecdd0084a07f8c97986c43af5c37ed03dd5aa994da6735cf59806162d1b2", + "solution/solve.sh": "3da9dbf589656512cd7a7004dc3752d3552a6af939cfe59eff79f91695fa42c7", + "task.toml": "b90a47ec26c1191d3c523cc094ea96b170e42fbfe176ef1f814c57d05390c097", + "tests/reference_illum1.png": "bb084f430bbc9d5b623a656920670dc06cee6d9132a9d550be4a1b99e41ab9e0", + "tests/test.sh": "928d1497ad21aef0c456294e3784f1fddf8493410f23584da89f9e02ec6bd134", + "tests/test_outputs.py": "64d6ce107503714ada346e5606832ee886be9950971cf81523e73d8ba28e8d37" + } + }, + { + "name": "terminal-bench/cobol-modernization", + "ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88", + "selection_rationale": "Legacy-language migration and behavioral compatibility", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "software-engineering", + "declared_difficulty": "easy", + "docker_image": "alexgshaw/cobol-modernization:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "c89318cf59efcc7040e3d946022e2616161b6733931a080cdeea7b1918e7ecc4", + "environment/Dockerfile": "8b175c83fc03ec7d16d6101516bc5100230b592f480e435279f4fe051c9280b2", + "environment/data/ACCOUNTS.DAT": "46f63eb29442ebc6716bcd8a0c36f4367910f2783f6b84a307ed9a61ccd1046e", + "environment/data/BOOKS.DAT": "db04da646207cd87f8e9c5b6f0b57d95953899f3eef22de1a81423f06ef4a9fd", + "environment/data/TRANSACTIONS.DAT": "1fa4cc5420ded94d585c563e8196b85166ebbae2bb8924b20014717fb2fb9ee3", + "environment/src/INPUT.DAT": "52fc984f0cf14af1495285d7efe02b9bae13159eea59255560ee938c4f78afee", + "environment/src/program.cbl": "27cb0e56f65f52fa75f346720b37a180b8e034ebe7d7622b979f195087bd6ebb", + "instruction.md": "a87074f1141b7345394cd88053d0debb0dc8f7de95308882c6538ee9a3b359da", + "solution/solve.sh": "7d4a0504e1686c1790bfbc074d4e598f4c631f54c2bc5f3d5fd786574b4810b9", + "task.toml": "4b620211dc3d6b1b989f0210c5ecd2f606974581c8bce27bd78659d48cf0cfbd", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "009180a7092583fc8143bb3ffc3957923aa0d61d0aac3a471f76eeb4aae54a88" + } + }, + { + "name": "terminal-bench/custom-memory-heap-crash", + "ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e", + "selection_rationale": "Native memory management debugging", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 1800, + "work_budget_seconds": 1620, + "verifier_timeout_seconds": 1800.0, + "category": "debugging", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/custom-memory-heap-crash:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "a06ed23a2c467aa36d0d4a88e5c020b1076cc25b8d483517239658e9c596dde9", + "environment/Dockerfile": "6ee81e5a4fa2540443d7f27d577ba840728fc6aab9f301ab7919103673c78080", + "environment/patches/locale_classes.tcc.patch": "16951f4a4df6bf253886ac75f5a39fdbc328e415194493f63cb7589a364aa817", + "environment/patches/locale_init.cc.patch": "e97f133ceebd86f8881f972e0e45b0ec0a13a9bfa428d8032008707683d54292", + "environment/program/main.cpp": "b94538c76887cc2b2ab739b5ac4731cddd0faa8f4a9892a6265c062bbbeede14", + "environment/program/user.cpp": "92b203786fed1bf0b3aa1675c9cd2d82b2651170675064da8ffee73b98ef14f9", + "environment/program/user.h": "f43d155995a56c22b9cd1417daef7c3b5a976cdc662a98eef0feb438ae1eee77", + "instruction.md": "53c68ae14a3c3af9a97c40bfd9bdc30a5d98b37eaa2f69f763475745f70f108d", + "solution/solve.sh": "b0b1d10bea1cd5be14a82b35d522df616df1e67028f90d2a6bef90aaa98511a1", + "task.toml": "ed474be1983e212913aa8866ebb66216acbe36aef2339ac375d232aa46617481", + "tests/test.sh": "70a2b55f0c5757f6b123a216dfd202be4a646680bced28dddcd31fd8959a058f", + "tests/test_outputs.py": "4e1cb33f777fa7fb5e1536f9bea9eace6deccd515c67d4f87570adbe692914ba" + } + }, + { + "name": "terminal-bench/fix-code-vulnerability", + "ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71", + "selection_rationale": "Security bug repair and behavior preservation", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "security", + "declared_difficulty": "hard", + "docker_image": "alexgshaw/fix-code-vulnerability:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "87f28dbad183c3a634887696d328ca68a8a0858f132bc48cb7341c22267d38f7", + "environment/Dockerfile": "0917470986224df4daf4854a71302ae81c73ba5dd6e25be1232d29b758a5b3ab", + "instruction.md": "89b1abdf0af19399f720233b5cdf6b2e8e0ff7b6e3fe3e2a3af0d4d298a8641a", + "solution/solve.sh": "7a0131b369d83ae8546ecc3f9b4e23d38240f1bd1f0941e4430fa85b5182ee03", + "task.toml": "9aa2bd14ce9408289d199d0608bc3fa65b8b5e23374c9641020de62197dbf1e2", + "tests/test.sh": "f1611da693cb4cfc5e8482698c6a1858a5c67220ae9db89b50aed66584e9b17f", + "tests/test_outputs.py": "c006a5de95b69bd78055aa26cc055734dbd50718ede25d331f767bfee4ef7f26" + } + }, + { + "name": "terminal-bench/nginx-request-logging", + "ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097", + "selection_rationale": "Service configuration and observable request behavior", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "system-administration", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/nginx-request-logging:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d79997586d1c14ed8dbfea050e3580b25500e04c8345acfd3f294152433c795a", + "environment/Dockerfile": "43003466d89ab83484a3fa618e35b58cdb74a002b1e8716ef30f12b3ac8302d8", + "instruction.md": "26a7ac98aced6107f147206790ada77bbf8ed3c25fb360cf323d5fc6889edf99", + "solution/solve.sh": "2454bbbf7c201a9a778165f286373ba2cb991a0eb1d026a8005d57ae71250dd9", + "task.toml": "b7a78d00be13656f75069197100b7b8c4fd268decf6f78929c853b3222a2729d", + "tests/test.sh": "27761195f53cbfe5941fe39b573631f6c926869e64b7c43220377e1e116528e7", + "tests/test_outputs.py": "045cc716c14efde3b0dcff5fc7c85ec5d18bfc6ce66f8b40a418fa2a3a4acda0" + } + }, + { + "name": "terminal-bench/git-multibranch", + "ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa", + "selection_rationale": "Cross-branch repository operations", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "system-administration", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/git-multibranch:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "3b9473429381d9eacabab498cb4a8915aa2bca24fa857ef862e5991d3f81f113", + "environment/Dockerfile": "88d9a9f4fef1ae7fddf632799dedf5a2fb6489460a93b610c33dd8b122982821", + "environment/default.conf": "a3a4c5d0ac9cfe79b0510e4225c80fa3988c27fbac13622505886bf66449e04a", + "instruction.md": "54014a38d7526d65faeb6c11d9b7ed3d4a5cb5d973d68d3da3b68b8c0289e125", + "solution/solve.sh": "300ca96b79db8d77f14742109aaf76048aafc4644ced750b1bc5c2761e8d0175", + "task.toml": "4b9981604578a31205257cca36a28d0c89c9d8d1058541330acd11d3f8d65f82", + "tests/test.sh": "ddbde3eea6ffa8fda105cbd86597a53bd409f41e13223900aa27e77a84b6b155", + "tests/test_outputs.py": "f03c6867663b8e8fb7899df51ee4df26411fab62ed491753964b801e591d16c5" + } + }, + { + "name": "terminal-bench/db-wal-recovery", + "ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4", + "selection_rationale": "Database recovery and data integrity", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "file-operations", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/db-wal-recovery:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "600762031a1a02ecc3b46d70f7683f3ff930b47c8a3d0e27cc31b146158c3126", + "environment/Dockerfile": "63a260531c62f4ae72060db17f295c0aa03cfc6321f8162c324b7aa6b3f81302", + "environment/main.db": "68cb288eb29e76d814b4e7ee6887b37c415675e644f88173bc191348e290d119", + "environment/main.db-wal.encrypted": "68753bdce26196a53194260d001544cc6bb8f9f38e2f2f6826d09798e31e6e26", + "instruction.md": "5a20375162921b8911c67035d1905df7fd335d1bed03044271e608b2c3c93b26", + "solution/solve.sh": "ed8530c94103bf19783a1d41bc63fe4db478cdd9629f22fee25c45a45699f5e8", + "task.toml": "1cdc2c615e259bc10265efbe71b518332070c1c002e226566399e9788f18d0aa", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "88dc5c2a3abba75bca0e0f9136223ae5bb585d5984b8d9eba51a7829c1d2ba49" + } + }, + { + "name": "terminal-bench/multi-source-data-merger", + "ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49", + "selection_rationale": "Data integration and cross-input consistency", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "data-processing", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/multi-source-data-merger:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "9bf37162a4c1b88ed93627830bb8512ebae154325d7cb069c883144a0e4c69d6", + "environment/Dockerfile": "c95a40d3d35b473dc15ae1e8784ba7fb1725805c0583fc574dd2764325d75e4f", + "environment/data/source_a/users.json": "75e51e2d3ab0bdba2269d7cf3757576f84f3ee2c13407d2db94289a5958a1778", + "environment/data/source_b/users.csv": "8e87d16dc47d166edd8e56165955bad265b630d53d02762367c816affb5fa760", + "environment/data/source_c/users.parquet": "23d1ea51c4170bdba53febb885a0f495e28de525a52667347b2932136b9a72fe", + "instruction.md": "d1d28bb6e37f7a255a1b7f78aae8591f35c0494f33e4913392424ad954aab944", + "solution/solve.sh": "1b3d0d50e31930d042495d48f4c8e95f1a9c1a979c803b7f7cec651322537e1e", + "task.toml": "ee3964c0ee040f0a0cb121cefefbc15443d127684acb003fa5b61dd42f31c694", + "tests/test.sh": "d5a00e64413c0e1ae452f38a1aa1a47c3ed7b4416bbfccae1a4c053025a336d1", + "tests/test_outputs.py": "a070888930cf829aeca039771260b1386bda990ea805f313468d381ba3608ca5" + } + }, + { + "name": "terminal-bench/query-optimize", + "ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4", + "selection_rationale": "Database query performance and correctness", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 1800.0, + "category": "data-science", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/query-optimize:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "a82c53e333a0ac0b0b410b5e26cb8e7305dece31db84d340248ffb13d5a45160", + "environment/Dockerfile": "064818771f200e0517d22068f87229a7a95d0a82b0778c84ca7203bf1926a0b8", + "environment/task-deps/my-sql-query.sql": "43382792f0f438625d643e34abe144febd5669518f4ab1ab550a15c6d9314a21", + "instruction.md": "473e12d320d2732f679fb631c2a4dab4a01deea6775686718ae0c417b8a39447", + "solution/solve.sh": "0d95f3cedb287ff9d47c796cbaf812c300d1222c10fa80c49e8acb0e81eb6978", + "task.toml": "6e88f2a86df699594bcb1bf81df4ed694505f79c9420f671335b02524bbe3868", + "tests/golden.sql": "4da1885d56b3ffa7b7b22b422996d018795611f6963499d551582881b74b84fd", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "2068ae80c57af8d52a7ae510f0666732c057806e556b480970f36f14bae683f6" + } + }, + { + "name": "terminal-bench/adaptive-rejection-sampler", + "ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de", + "selection_rationale": "Numerical sampling and statistical invariants", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "scientific-computing", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/adaptive-rejection-sampler:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "95408217b9c484ffc18a02a91b7706323b6902919c72b24866a32e27a23543e2", + "environment/Dockerfile": "e644a3343e339303230f1441b7383f076d21c69a5a7aa6430b4f8a9d27f15c8b", + "environment/protected.tar.gz.enc": "7fc9bf1347bb97da8dede04b2c58638f225070913e222d76627068aebd1b1df2", + "instruction.md": "e6d28c28acd76bef61817427c74c2a29e3e8d75f535cbd708afcbfd02c54013f", + "solution/solve.sh": "fe443a8082b5183ba1623311ba0ee958888aefa633eff93766b9ae14e9bdd84d", + "task.toml": "6ed7b37f74beb46e9d47b0f5fad53025667c9548baff4e96d467b56d429fab1a", + "tests/test.sh": "38b43560d173cc2b952c3a3e17b8a480216d84e33450515e047bcb0d806b1e0a", + "tests/test_outputs.py": "547dc6e107f034f41703722aeceb6d0236e3fb69116fc3f2fbaba11884de352f" + } + }, + { + "name": "terminal-bench/constraints-scheduling", + "ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53", + "selection_rationale": "Constrained algorithmic planning", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 1200, + "work_budget_seconds": 1020, + "verifier_timeout_seconds": 1200.0, + "category": "personal-assistant", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/constraints-scheduling:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "d54a690970149e1216bd4fc066b45069c7948c13e3b1ef2a213a74342805f209", + "environment/Dockerfile": "7d9aba072a4a907229728b9739c8cf821fc2c98450babd64452682caf3759c84", + "environment/inputs/alice_calendar.ics": "5a5d45804bffd339c2479d49e76fe817b38244d14d389afe1de583443808ecbc", + "environment/inputs/bob_calendar.ics": "c97e85dbcdd756758f8a80f4db5fd5c1dcacee79298e4d2262f46de5ff680b50", + "environment/inputs/carol_calendar.ics": "32a0f8687637df0a12b8e5c56d49c4192bc3e790856943632d70f237c7c1363b", + "instruction.md": "342def78177f9bc41bea4d11af679bc385a9dda2ac8a1e425be384a5b7f937b6", + "solution/solve.sh": "802d81ec064fad3050bbade95a2f847ac24e0b7f80fa69b0bcee0347dc92fe77", + "task.toml": "7b9842de16d74e8faf8c7c0aa03b0b81145941cb12dd322d81ae23fc48607250", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "1422b345ba3aacdfc4184e962811b6379988435896c50d8cf62903e70990c5d1" + } + }, + { + "name": "terminal-bench/pytorch-model-cli", + "ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79", + "selection_rationale": "Model packaging and command-line contracts", + "cohort": "new_development_baseline", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "category": "model-training", + "declared_difficulty": "medium", + "docker_image": "alexgshaw/pytorch-model-cli:20251031", + "resources": { + "cpus": 1, + "memory_mb": 2048, + "storage_mb": 10240, + "gpus": 0 + }, + "verifier_dependency_profile": "not_configured", + "task_file_sha256": { + "README.md": "35a093fa5f1592737784ba718c2e3e9ae17ff8b5e53e003fae560ddce091039c", + "environment/Dockerfile": "8fd5c0f21a6914cd440d10c84c9db06fec606f20dc3e0363b17a3b8c220dc4d9", + "environment/task-deps/cJSON.c": "0e7995261640c96db16ed2c91a5d981efa917cf5e72d0f97ec45a1179ae3ad73", + "environment/task-deps/cJSON.h": "0909160541bcc510e0b2d8c7a9cc051fa1459a0f13be4d8e56346a15491f2f2f", + "environment/task-deps/image.png": "f4d5dfefe463939996541404340e7099d5957b56e1df3760d13b5d2de754db53", + "environment/task-deps/lodepng.cpp": "67873c5f6272f779a143a9c81070e8783327c5552d0b9de2aaee2cb779308917", + "environment/task-deps/lodepng.h": "b735b7e0357319973e9488d520d7f270d734a6e9e1bf924fb1d5b8d26d7e97ce", + "environment/task-deps/model.pth": "a32103eef09936e586b1a7dc7e64d279500a38ff30fdd6df2167e28eb954eee6", + "environment/task-deps/model.py": "e4d3a5c728449656d0efde7871ee227619348352769795ed1be1a8b7f835fddc", + "instruction.md": "ce36ee1f6c589f81ee6a6be4967e25c76a2ed697006928023f42dfb9f28a6750", + "solution/solve.sh": "3b34e561c9ef0a53c9948655847846eb4811aaac67b990480aa6aac1b04bfe66", + "task.toml": "fc1c018efbd5d0537ac0741d53d1ebad3886faa51d14cbde8ec953856411bc0d", + "tests/test.sh": "43fb67e537c7adc2df11c2367f47c876cb5d16a7c1ae16d0339c3a3364bb2b06", + "tests/test_outputs.py": "370702c57a4e0c85f70ee37a5c357ae3a5df8480a275bb6263b0376f2f6c093b" + } + } + ], + "total_model_runs": 12, + "max_concurrent_trials": 2, + "whole_trial_retries": 0, + "preflight": [], + "queue": [ + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/cobol-modernization", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/cobol-modernization.json", + "job_name": "dev12-cobol-modernization-baseline-20261011", + "config_sha256": "2680126c2f162d58948cd30565806bf6cb1f026d561c87b942ba4735173ea6a2" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/fix-code-vulnerability", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/fix-code-vulnerability.json", + "job_name": "dev12-fix-code-vulnerability-baseline-20261011", + "config_sha256": "4b2de43b547ddd92d748f7658c44b98b4fef25efd406710ace6d1713b9c789a7" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/nginx-request-logging", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/nginx-request-logging.json", + "job_name": "dev12-nginx-request-logging-baseline-20261011", + "config_sha256": "4a79d4fc4431acd942548f36188c1820ac5406fc6a1385cf505daf02aa190da3" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/git-multibranch", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/git-multibranch.json", + "job_name": "dev12-git-multibranch-baseline-20261011", + "config_sha256": "ae9c6d5830c9dcb8aab8b1356441b47731957e2ad5e5747327b7caa1a6c037a1" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/db-wal-recovery", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/db-wal-recovery.json", + "job_name": "dev12-db-wal-recovery-baseline-20261011", + "config_sha256": "475b7df31549e51f04f4f9171332da5a8ba03f7fbec2b3788fbf897340c0c91b" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/multi-source-data-merger", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/multi-source-data-merger.json", + "job_name": "dev12-multi-source-data-merger-baseline-20261011", + "config_sha256": "5eac4294149a11f037f732a32e1a8874613d8a28884843955481a9e8ef10b7bd" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/query-optimize", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/query-optimize.json", + "job_name": "dev12-query-optimize-baseline-20261011", + "config_sha256": "29ad0478282cd6fb86a1b209a93ffe33eee724c321916732db001b7beef8ea7e" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/adaptive-rejection-sampler", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/adaptive-rejection-sampler.json", + "job_name": "dev12-adaptive-rejection-sampler-baseline-20261011", + "config_sha256": "e0fec8e3f1f8e077a5384058ea5b6e4733973a7475b02556f90cfc0f4ba96f6d" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/pytorch-model-cli", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/pytorch-model-cli.json", + "job_name": "dev12-pytorch-model-cli-baseline-20261011", + "config_sha256": "bc915a7de1286954473271e2232f9c83af983c23b885d099316b33dc3c1d445d" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/constraints-scheduling", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/constraints-scheduling.json", + "job_name": "dev12-constraints-scheduling-baseline-20261011", + "config_sha256": "33813c2fd5b45efd3b7bf8a765fcb1f74c702821a3d4f473e4b479200433734e" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/custom-memory-heap-crash", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/custom-memory-heap-crash.json", + "job_name": "dev12-custom-memory-heap-crash-baseline-20261011", + "config_sha256": "4678ab6961d82a2d597911bd8d4a53040a4be526c1e97afb27c614ecedb1c05f" + }, + { + "arm": "main", + "repeat": 1, + "task": "terminal-bench/build-pov-ray", + "config": "benchmarks/harbor/configs/dev12-baseline-20261011/build-pov-ray.json", + "job_name": "dev12-build-pov-ray-baseline-20261011", + "config_sha256": "42e1bb7d2a1a6d5cf9f059a121140f9135d8b4632482a00d767a9aba2e1ac2e7" + } + ], + "log_dir": "jobs/dev12-baseline-20261011-v3-orchestration", + "budget_policy": "Native task agent timeout minus 180 seconds for runtime work; setup separately allows 1800 seconds. Native verifier limits remain unchanged. Build POV-Ray retains its unusually long 12000-second native limit (11820-second model work).", + "execution_policy": "One run per selected task. Run all twelve installation preflights before models. Queue is fixed by ascending native work budget, stable within ties; run consecutive pairs with at most two concurrent model trials.", + "failure_policy": "Retain every selected trial including setup failures, verifier errors and scored failures; no outcome-based substitutions or model reruns. Only verifier timeouts may retry, at most three total attempts on the same output with the native per-attempt limit.", + "preflight_policy": "All selected verifier dependency profiles are not_configured. Agent installation success does not certify verifier dependencies. Registered profiles, if present, must pass before model work.", + "information_policy": "Selection used task names before downloading. Only public metadata/instructions may be inspected before every agent output freezes; task files are hashed as bytes without inspecting hidden test or solution contents. Audit external references afterwards.", + "scope": "Development baseline for twelve new tasks, not a representative full-dataset score or a pass-rate improvement experiment. Existing 22 task records remain separate; remaining 55 tasks are not opened.", + "protocol_amendment": { + "supersedes": "experiment-v2.json", + "supersedes_sha256": "87f3c760051997e8957ae12efc77085040885053f44b4a9490565e4d3b371647", + "evidence_file": "preflight-v2-evidence.json", + "evidence_sha256": "8b4edad67647f78f4dd63ffafebee68bfbc5cfa19af91b8ded898826d8c14d46", + "previous_evidence_file": "preflight-v1-evidence.json", + "previous_evidence_sha256": "c723c858571a512e968db16babed482a2874f270561ae3ae8c426a2ad74bd73f", + "reason": "Use eleven successful installation-only trials from the second preflight plus the successful POV-Ray installation from the first. Preserve both failed job results and every failed trial. Each queued task/arm pair needs exactly one successful native installation-only result, bound to task revision, complete source snapshot, installed wheel, bootstrap, input configuration and artifact hashes. No model work has started.", + "changed_fields": [ + "registered_at", + "log_dir", + "preflight", + "accepted_preflight_trials", + "protocol_amendment" + ], + "unchanged": "Task selection and revisions, agent and adapter source, prompt, model, all task configurations and budgets, model queue and job names, concurrency, retry and information policies.", + "model_runs_started_at_amendment": 0 + }, + "accepted_preflight_trials": [ + { + "arm": "main", + "task": "terminal-bench/build-pov-ray", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-20261011/build-pov-ray__JBS6iy8/result.json", + "result_sha256": "c878116d4f2746c3269ceefba34b9f0d1c0d9672d3010adb43ff904767391c84", + "bootstrap_sha256": "40256d355dfe43492278a1a4a22b8a6093657509086254ddd27494162c85b0b4", + "manifest_sha256": "b36023c8c2ba8248ae41ab708b1bd39a8cb99f665522960170e51f20d1bb6227", + "input_config_sha256": "6dc58db5abd850d91f1e64d0b35a2a82ed85a36ffdff763b93ff60e5b050081a" + }, + { + "arm": "main", + "task": "terminal-bench/cobol-modernization", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/cobol-modernization__2A6Ldop/result.json", + "result_sha256": "f1010dc85724456c52f48644016afdf15a2a29dc68c57430270a01d16f75813b", + "bootstrap_sha256": "1812b1d7452b4ec930ab65be85cd8ba7f2e96342153c5e9e7d0dc948f342502f", + "manifest_sha256": "7209a44f49fe68e5059dd8d36823546445ae8b86b7abc8c098ee0f6eaa0199f0", + "input_config_sha256": "901854a68691bc24f5d95b37f0d4d292f970ca691ee3e1f7a7f8c7ffe0afca18" + }, + { + "arm": "main", + "task": "terminal-bench/custom-memory-heap-crash", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/custom-memory-heap-crash__D2vB9wp/result.json", + "result_sha256": "2151d83279d3fea4a998d1611cf7e523ae061bd1592ddb6df3825939d0477c1c", + "bootstrap_sha256": "cf6bb84c42557ba2a7b7e0ec653bbc1c7cf86c236d1b7e2a6852918dfb8eb7dc", + "manifest_sha256": "7209a44f49fe68e5059dd8d36823546445ae8b86b7abc8c098ee0f6eaa0199f0", + "input_config_sha256": "901854a68691bc24f5d95b37f0d4d292f970ca691ee3e1f7a7f8c7ffe0afca18" + }, + { + "arm": "main", + "task": "terminal-bench/fix-code-vulnerability", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/fix-code-vulnerability__m7qrvoe/result.json", + "result_sha256": "4914a240602a36fedab3ac6313643cb1beffc22de6724fd48b45f4974efa0339", + "bootstrap_sha256": "8d1fa3e359c1a2c87bb9b262c9113a423af3445aa5a20da8a2e6fb4d63b4f67a", + "manifest_sha256": "7209a44f49fe68e5059dd8d36823546445ae8b86b7abc8c098ee0f6eaa0199f0", + "input_config_sha256": "901854a68691bc24f5d95b37f0d4d292f970ca691ee3e1f7a7f8c7ffe0afca18" + }, + { + "arm": "main", + "task": "terminal-bench/nginx-request-logging", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/nginx-request-logging__LpiAC8p/result.json", + "result_sha256": "edc76f020a59073905b3d74d47b0bca454a3b5a43b13c0df23fc19b1bf0e0bd2", + "bootstrap_sha256": "bd617a3903bf50c818febe3abf77cc268952deb3cd9f0c6db99bc0cbf8146a94", + "manifest_sha256": "7209a44f49fe68e5059dd8d36823546445ae8b86b7abc8c098ee0f6eaa0199f0", + "input_config_sha256": "901854a68691bc24f5d95b37f0d4d292f970ca691ee3e1f7a7f8c7ffe0afca18" + }, + { + "arm": "main", + "task": "terminal-bench/git-multibranch", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/git-multibranch__NsTYkJt/result.json", + "result_sha256": "66a667c7278b36d1bee52e1bf466d1464c7debdde77a288f0fb62d1ea7fbbd6f", + "bootstrap_sha256": "391e000959d10d8aa551310ebe7bddd0c70185a684ee8cb543b7e8bf57e5e63b", + "manifest_sha256": "7209a44f49fe68e5059dd8d36823546445ae8b86b7abc8c098ee0f6eaa0199f0", + "input_config_sha256": "901854a68691bc24f5d95b37f0d4d292f970ca691ee3e1f7a7f8c7ffe0afca18" + }, + { + "arm": "main", + "task": "terminal-bench/db-wal-recovery", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/db-wal-recovery__pJKJsai/result.json", + "result_sha256": "c0f6d3334e4f8bedd61a811ac5110a1776bf9a9d56bd1037910501d552b70289", + "bootstrap_sha256": "2ffe3efbf9fd9f423c3fc82102152f59af7f9180f8aaf39c46e61c96e2f02d76", + "manifest_sha256": "7209a44f49fe68e5059dd8d36823546445ae8b86b7abc8c098ee0f6eaa0199f0", + "input_config_sha256": "901854a68691bc24f5d95b37f0d4d292f970ca691ee3e1f7a7f8c7ffe0afca18" + }, + { + "arm": "main", + "task": "terminal-bench/multi-source-data-merger", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/multi-source-data-merger__3LWnqvT/result.json", + "result_sha256": "da74a33cea03bc9a54f5ff3e39c1ed8a3c07a9ed1ae7405ff73069c68f1a782c", + "bootstrap_sha256": "49ba4308d822fabb4ca040d48e8e24e7c3bd34137bc08e64c1e19b856268814c", + "manifest_sha256": "7209a44f49fe68e5059dd8d36823546445ae8b86b7abc8c098ee0f6eaa0199f0", + "input_config_sha256": "901854a68691bc24f5d95b37f0d4d292f970ca691ee3e1f7a7f8c7ffe0afca18" + }, + { + "arm": "main", + "task": "terminal-bench/query-optimize", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/query-optimize__ng6Cxoz/result.json", + "result_sha256": "0bdc5b650976ab994bea028e62b542682f8e9f13005fba846797f8ec11b8b580", + "bootstrap_sha256": "0a52c1c56dbec333ab809d00b69469021e06f49e0570249b280b6aacef2587f0", + "manifest_sha256": "7209a44f49fe68e5059dd8d36823546445ae8b86b7abc8c098ee0f6eaa0199f0", + "input_config_sha256": "901854a68691bc24f5d95b37f0d4d292f970ca691ee3e1f7a7f8c7ffe0afca18" + }, + { + "arm": "main", + "task": "terminal-bench/adaptive-rejection-sampler", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/adaptive-rejection-sampler__tXynuKz/result.json", + "result_sha256": "92c4b295be0b5890374c4b124ba339b19111969252a670547a5d47a46047d878", + "bootstrap_sha256": "eb4895cdc7076ee2da4b9519eb1d623465f7c0828b9afb96a5d685f89a8c1f1b", + "manifest_sha256": "7209a44f49fe68e5059dd8d36823546445ae8b86b7abc8c098ee0f6eaa0199f0", + "input_config_sha256": "901854a68691bc24f5d95b37f0d4d292f970ca691ee3e1f7a7f8c7ffe0afca18" + }, + { + "arm": "main", + "task": "terminal-bench/constraints-scheduling", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/constraints-scheduling__sEgWjQv/result.json", + "result_sha256": "9c845d39b1d6ee7db05df413559408a68f99711d9a89754fe1ea474fb72cb1d3", + "bootstrap_sha256": "c4d5399fb99e22fdaccbafaeca054f4d615062b69ddc2389af0ed1c19ba30325", + "manifest_sha256": "7209a44f49fe68e5059dd8d36823546445ae8b86b7abc8c098ee0f6eaa0199f0", + "input_config_sha256": "901854a68691bc24f5d95b37f0d4d292f970ca691ee3e1f7a7f8c7ffe0afca18" + }, + { + "arm": "main", + "task": "terminal-bench/pytorch-model-cli", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/pytorch-model-cli__Gm2gHNY/result.json", + "result_sha256": "4c3ba76c5299e13b0be55de1f63ec981933e126136026e2ce06fbbfe4d6811cd", + "bootstrap_sha256": "9abae99e4800f37de01cc8c2eb9d13d0437c60d0c1c0724f648d551c7f382b77", + "manifest_sha256": "7209a44f49fe68e5059dd8d36823546445ae8b86b7abc8c098ee0f6eaa0199f0", + "input_config_sha256": "901854a68691bc24f5d95b37f0d4d292f970ca691ee3e1f7a7f8c7ffe0afca18" + } + ] +} diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/preflight-v2-evidence.json b/benchmarks/harbor/configs/dev12-baseline-20261011/preflight-v2-evidence.json new file mode 100644 index 0000000..c1e99fe --- /dev/null +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/preflight-v2-evidence.json @@ -0,0 +1,206 @@ +{ + "recorded_at": "2026-10-11T15:48:27.483536+00:00", + "preflight_state_path": "jobs/dev12-baseline-20261011-v2-orchestration/dev12-main-install-only-v2-20261011.json", + "preflight_state_sha256": "8f4fb628bbee4faae150ef1932dff53a2a90d695c8d08fd551c72a6b93af82cd", + "preflight_exit_code": 1, + "summary": { + "setup_passed": 11, + "agent_install_network_errors": 1, + "model_runs_started": 0 + }, + "failure_diagnosis": "POV-Ray: curl returned 18 (transferred a partial file) while downloading the original uv 0.9.11 release archive. Its earlier installation succeeded.", + "trials": [ + { + "task": "terminal-bench/adaptive-rejection-sampler", + "trial": "adaptive-rejection-sampler__tXynuKz", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/adaptive-rejection-sampler__tXynuKz/result.json", + "result_sha256": "92c4b295be0b5890374c4b124ba339b19111969252a670547a5d47a46047d878", + "exception_type": null, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/adaptive-rejection-sampler__tXynuKz/agent/bootstrap.json", + "bootstrap_sha256": "eb4895cdc7076ee2da4b9519eb1d623465f7c0828b9afb96a5d685f89a8c1f1b", + "bootstrap_status": "ready", + "bootstrap_stage": "complete", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "model_started": false + }, + { + "task": "terminal-bench/build-pov-ray", + "trial": "build-pov-ray__55Ff9nd", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/build-pov-ray__55Ff9nd/result.json", + "result_sha256": "c3e7244e688ad45a31f8bafccdab1e9dccef1539189d9674640b37e7810681ac", + "exception_type": "NetworkConnectionError", + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/build-pov-ray__55Ff9nd/agent/bootstrap.json", + "bootstrap_sha256": "f906cdc7af870689cc72b31c415d66f9788fce687ea0f6b40d3b5e9e66e0fe1a", + "bootstrap_status": "failed", + "bootstrap_stage": "agent_install", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "model_started": false + }, + { + "task": "terminal-bench/cobol-modernization", + "trial": "cobol-modernization__2A6Ldop", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/cobol-modernization__2A6Ldop/result.json", + "result_sha256": "f1010dc85724456c52f48644016afdf15a2a29dc68c57430270a01d16f75813b", + "exception_type": null, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/cobol-modernization__2A6Ldop/agent/bootstrap.json", + "bootstrap_sha256": "1812b1d7452b4ec930ab65be85cd8ba7f2e96342153c5e9e7d0dc948f342502f", + "bootstrap_status": "ready", + "bootstrap_stage": "complete", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "model_started": false + }, + { + "task": "terminal-bench/constraints-scheduling", + "trial": "constraints-scheduling__sEgWjQv", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/constraints-scheduling__sEgWjQv/result.json", + "result_sha256": "9c845d39b1d6ee7db05df413559408a68f99711d9a89754fe1ea474fb72cb1d3", + "exception_type": null, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/constraints-scheduling__sEgWjQv/agent/bootstrap.json", + "bootstrap_sha256": "c4d5399fb99e22fdaccbafaeca054f4d615062b69ddc2389af0ed1c19ba30325", + "bootstrap_status": "ready", + "bootstrap_stage": "complete", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "model_started": false + }, + { + "task": "terminal-bench/custom-memory-heap-crash", + "trial": "custom-memory-heap-crash__D2vB9wp", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/custom-memory-heap-crash__D2vB9wp/result.json", + "result_sha256": "2151d83279d3fea4a998d1611cf7e523ae061bd1592ddb6df3825939d0477c1c", + "exception_type": null, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/custom-memory-heap-crash__D2vB9wp/agent/bootstrap.json", + "bootstrap_sha256": "cf6bb84c42557ba2a7b7e0ec653bbc1c7cf86c236d1b7e2a6852918dfb8eb7dc", + "bootstrap_status": "ready", + "bootstrap_stage": "complete", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "model_started": false + }, + { + "task": "terminal-bench/db-wal-recovery", + "trial": "db-wal-recovery__pJKJsai", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/db-wal-recovery__pJKJsai/result.json", + "result_sha256": "c0f6d3334e4f8bedd61a811ac5110a1776bf9a9d56bd1037910501d552b70289", + "exception_type": null, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/db-wal-recovery__pJKJsai/agent/bootstrap.json", + "bootstrap_sha256": "2ffe3efbf9fd9f423c3fc82102152f59af7f9180f8aaf39c46e61c96e2f02d76", + "bootstrap_status": "ready", + "bootstrap_stage": "complete", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "model_started": false + }, + { + "task": "terminal-bench/fix-code-vulnerability", + "trial": "fix-code-vulnerability__m7qrvoe", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/fix-code-vulnerability__m7qrvoe/result.json", + "result_sha256": "4914a240602a36fedab3ac6313643cb1beffc22de6724fd48b45f4974efa0339", + "exception_type": null, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/fix-code-vulnerability__m7qrvoe/agent/bootstrap.json", + "bootstrap_sha256": "8d1fa3e359c1a2c87bb9b262c9113a423af3445aa5a20da8a2e6fb4d63b4f67a", + "bootstrap_status": "ready", + "bootstrap_stage": "complete", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "model_started": false + }, + { + "task": "terminal-bench/git-multibranch", + "trial": "git-multibranch__NsTYkJt", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/git-multibranch__NsTYkJt/result.json", + "result_sha256": "66a667c7278b36d1bee52e1bf466d1464c7debdde77a288f0fb62d1ea7fbbd6f", + "exception_type": null, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/git-multibranch__NsTYkJt/agent/bootstrap.json", + "bootstrap_sha256": "391e000959d10d8aa551310ebe7bddd0c70185a684ee8cb543b7e8bf57e5e63b", + "bootstrap_status": "ready", + "bootstrap_stage": "complete", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "model_started": false + }, + { + "task": "terminal-bench/multi-source-data-merger", + "trial": "multi-source-data-merger__3LWnqvT", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/multi-source-data-merger__3LWnqvT/result.json", + "result_sha256": "da74a33cea03bc9a54f5ff3e39c1ed8a3c07a9ed1ae7405ff73069c68f1a782c", + "exception_type": null, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/multi-source-data-merger__3LWnqvT/agent/bootstrap.json", + "bootstrap_sha256": "49ba4308d822fabb4ca040d48e8e24e7c3bd34137bc08e64c1e19b856268814c", + "bootstrap_status": "ready", + "bootstrap_stage": "complete", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "model_started": false + }, + { + "task": "terminal-bench/nginx-request-logging", + "trial": "nginx-request-logging__LpiAC8p", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/nginx-request-logging__LpiAC8p/result.json", + "result_sha256": "edc76f020a59073905b3d74d47b0bca454a3b5a43b13c0df23fc19b1bf0e0bd2", + "exception_type": null, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/nginx-request-logging__LpiAC8p/agent/bootstrap.json", + "bootstrap_sha256": "bd617a3903bf50c818febe3abf77cc268952deb3cd9f0c6db99bc0cbf8146a94", + "bootstrap_status": "ready", + "bootstrap_stage": "complete", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "model_started": false + }, + { + "task": "terminal-bench/pytorch-model-cli", + "trial": "pytorch-model-cli__Gm2gHNY", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/pytorch-model-cli__Gm2gHNY/result.json", + "result_sha256": "4c3ba76c5299e13b0be55de1f63ec981933e126136026e2ce06fbbfe4d6811cd", + "exception_type": null, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/pytorch-model-cli__Gm2gHNY/agent/bootstrap.json", + "bootstrap_sha256": "9abae99e4800f37de01cc8c2eb9d13d0437c60d0c1c0724f648d551c7f382b77", + "bootstrap_status": "ready", + "bootstrap_stage": "complete", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "model_started": false + }, + { + "task": "terminal-bench/query-optimize", + "trial": "query-optimize__ng6Cxoz", + "result_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/query-optimize__ng6Cxoz/result.json", + "result_sha256": "0bdc5b650976ab994bea028e62b542682f8e9f13005fba846797f8ec11b8b580", + "exception_type": null, + "bootstrap_path": "jobs/dev12-baseline-main-source/jobs/dev12-main-install-only-v2-20261011/query-optimize__ng6Cxoz/agent/bootstrap.json", + "bootstrap_sha256": "0a52c1c56dbec333ab809d00b69469021e06f49e0570249b280b6aacef2587f0", + "bootstrap_status": "ready", + "bootstrap_stage": "complete", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "model_started": false + } + ] +} diff --git a/benchmarks/harbor/scripts/run_comparison.py b/benchmarks/harbor/scripts/run_comparison.py index 9266f1f..1f2cfbc 100644 --- a/benchmarks/harbor/scripts/run_comparison.py +++ b/benchmarks/harbor/scripts/run_comparison.py @@ -114,6 +114,67 @@ def preflight_passed(states, expected_counts): ) +def accepted_preflights_passed(root, plan): + """Validate previously completed installation trials without replaying jobs. + + A failed sibling trial does not invalidate a successful installation. Every + task/arm pair still needs native, hash-bound evidence from the frozen source. + This mode cannot be combined with new preflight jobs in the same plan. + """ + if plan["preflight"]: + raise ValueError("Accepted installation evidence cannot mix with new preflight jobs") + expected = {(j["arm"], j["task"]) for j in plan["queue"]} + evidence = plan["accepted_preflight_trials"] + pairs = [(e["arm"], e["task"]) for e in evidence] + if not expected or set(pairs) != expected or len(pairs) != len(expected): + raise ValueError("Every queued task/arm pair needs exactly one accepted preflight") + tasks = {t["name"]: t for t in plan["tasks"]} + + def checked(path, expected_hash): + if digest(path) != expected_hash: + raise ValueError(f"Accepted preflight evidence changed: {path}") + return json.loads(path.read_text()) + + for item in evidence: + arm = plan["arms"][item["arm"]] + source = root / arm["source_root"] + result_path = (root / item["result_path"]).resolve() + if not result_path.is_relative_to((source / "jobs").resolve()) or result_path.name != "result.json": + raise ValueError("Accepted trial must belong to the frozen source checkout") + result = checked(result_path, item["result_sha256"]) + task = tasks[item["task"]] + task_id = result["task_id"] + if result["task_name"] != item["task"] or task_id["ref"] != task["ref"]: + raise ValueError("Accepted preflight has a different task revision") + if (not result.get("finished_at") or result.get("exception_info") + or result.get("agent_execution") is not None + or result.get("verifier") is not None + or result.get("verifier_result") is not None): + raise ValueError("Accepted evidence must be a successful installation-only trial") + bootstrap = checked(result_path.parent / "agent/bootstrap.json", item["bootstrap_sha256"]) + if (bootstrap.get("status") != "ready" or bootstrap.get("stage") != "complete" + or bootstrap.get("verifier_preflight", {}).get("status") not in {"passed", "not_configured"}): + raise ValueError("Accepted installation did not finish its required preflight") + job_dir = result_path.parent.parent + inputs = job_dir.with_name(job_dir.name + "-input") + manifest = checked(inputs / "manifest.json", item["manifest_sha256"]) + config = checked(inputs / "config.json", item["input_config_sha256"]) + if config.get("install_only") is not True or manifest["commit"] != arm["source_ref"]: + raise ValueError("Accepted installation used a different mode or source revision") + expected_sources = {} + for prefix, package in (("src", "nanopycodeagent"), ("benchmarks/harbor/src", "harbor_adapter")): + for path in (source / prefix / package).rglob("*.py"): + expected_sources[str(path.relative_to(source / prefix))] = digest(path) + if not expected_sources or manifest["source_sha256"] != expected_sources: + raise ValueError("Accepted installation source snapshot differs from the frozen source") + wheel = Path(result["config"]["agent"]["kwargs"]["wheel_path"]).resolve() + if not wheel.is_relative_to((inputs / "dist").resolve()): + raise ValueError("Accepted installation wheel must belong to its recorded inputs") + if not (digest(wheel) == manifest["wheel_sha256"] == bootstrap["wheel_sha256"]): + raise ValueError("Accepted installation wheel digest differs from its provenance") + return True + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--plan", type=Path, required=True) @@ -122,6 +183,10 @@ def main(): root, plan_path = Path.cwd(), args.plan.resolve() plan = json.loads(plan_path.read_text()) validate_plan(root, plan_path, plan) + accepted = ( + accepted_preflights_passed(root, plan) + if "accepted_preflight_trials" in plan else False + ) record = root / plan["log_dir"] record.mkdir(parents=True, exist_ok=True) record.chmod(0o700) @@ -138,10 +203,11 @@ def main(): ] if args.phase == "preflight": states = [run_job(root, plan, j, record, True) for j in plan["preflight"]] - return 0 if preflight_passed(states, expected_counts) else 1 - preflight = [json.loads((record / (j["job_name"] + ".json")).read_text()) for j in plan["preflight"]] - if not preflight_passed(preflight, expected_counts): - raise RuntimeError("Install-only checks must finish successfully before model work") + return 0 if accepted or preflight_passed(states, expected_counts) else 1 + if not accepted: + preflight = [json.loads((record / (j["job_name"] + ".json")).read_text()) for j in plan["preflight"]] + if not preflight_passed(preflight, expected_counts): + raise RuntimeError("Install-only checks must finish successfully before model work") states = [] with ThreadPoolExecutor(max_workers=2) as pool: for offset in range(0, len(plan["queue"]), 2): diff --git a/benchmarks/harbor/tests/test_comparison.py b/benchmarks/harbor/tests/test_comparison.py index 61faa56..7284900 100644 --- a/benchmarks/harbor/tests/test_comparison.py +++ b/benchmarks/harbor/tests/test_comparison.py @@ -55,3 +55,109 @@ def test_interrupted_job_cannot_silently_restart(tmp_path): (tmp_path / "interrupted.json").write_text(json.dumps({"status": "started"})) with pytest.raises(RuntimeError, match="refusing to retry"): comparison.run_job(tmp_path, {}, job, tmp_path, False) + + +@pytest.fixture +def accepted_installation(tmp_path): + source = tmp_path / "source" + module = source / "src/nanopycodeagent/agent.py" + module.parent.mkdir(parents=True) + module.write_text("# frozen runtime\n") + trial = source / "jobs/preflight/task__one" + (trial / "agent").mkdir(parents=True) + inputs = source / "jobs/preflight-input" + (inputs / "dist").mkdir(parents=True) + wheel = inputs / "dist/runtime.whl" + wheel.write_bytes(b"recorded wheel bytes") + files = { + "result": (trial / "result.json", { + "task_name": "org/task", "task_id": {"ref": "sha256:task"}, + "finished_at": "2026-10-11T00:00:00Z", "exception_info": None, + "agent_execution": None, "verifier": None, "verifier_result": None, + "config": {"agent": {"kwargs": {"wheel_path": str(wheel)}}}, + }), + "bootstrap": (trial / "agent/bootstrap.json", { + "status": "ready", "stage": "complete", + "verifier_preflight": {"status": "not_configured"}, + "wheel_sha256": comparison.digest(wheel), + }), + "manifest": (inputs / "manifest.json", { + "commit": "frozen", "wheel_sha256": comparison.digest(wheel), + "source_sha256": {"nanopycodeagent/agent.py": comparison.digest(module)}, + }), + "input_config": (inputs / "config.json", {"install_only": True}), + } + evidence = {"arm": "main", "task": "org/task", "result_path": str((trial / "result.json").relative_to(tmp_path))} + for name, (path, data) in files.items(): + path.write_text(json.dumps(data)) + evidence[name + "_sha256"] = comparison.digest(path) + plan = { + "preflight": [], "queue": [{"arm": "main", "task": "org/task"}], + "arms": {"main": {"source_root": "source", "source_ref": "frozen"}}, + "tasks": [{"name": "org/task", "ref": "sha256:task"}], + "accepted_preflight_trials": [evidence], + } + return plan, files, evidence, module, wheel + + +def test_accept_native_success_even_when_a_sibling_failed(tmp_path, accepted_installation): + plan, files, _, _, _ = accepted_installation + sibling = files["result"][0].parent.parent / "failed__two" + sibling.mkdir() + (sibling / "result.json").write_text(json.dumps({"exception_info": {"exception_type": "NetworkConnectionError"}})) + assert comparison.accepted_preflights_passed(tmp_path, plan) + assert (sibling / "result.json").exists() + + +@pytest.mark.parametrize("change", ["missing", "duplicate", "mixed"]) +def test_every_pair_needs_exactly_one_success(tmp_path, accepted_installation, change): + plan, _, evidence, _, _ = accepted_installation + if change == "missing": + plan["queue"].append({"arm": "main", "task": "org/second"}) + elif change == "duplicate": + plan["accepted_preflight_trials"].append(evidence.copy()) + else: + plan["preflight"].append({"job_name": "another"}) + with pytest.raises(ValueError): + comparison.accepted_preflights_passed(tmp_path, plan) + + +@pytest.mark.parametrize("field,value", [ + ("exception_info", {"exception_type": "AgentSetupTimeoutError"}), + ("agent_execution", {"started_at": "2026-10-11T00:00:00Z"}), + ("verifier_result", {"rewards": {"reward": 1}}), + ("finished_at", None), + ("task_id", {"ref": "sha256:different"}), +]) +def test_ineligible_trials_cannot_be_accepted(tmp_path, accepted_installation, field, value): + plan, files, evidence, _, _ = accepted_installation + path, data = files["result"] + data[field] = value + path.write_text(json.dumps(data)) + evidence["result_sha256"] = comparison.digest(path) + with pytest.raises(ValueError): + comparison.accepted_preflights_passed(tmp_path, plan) + + +@pytest.mark.parametrize("changed", ["result", "source", "wheel"]) +def test_evidence_and_installed_source_must_still_match(tmp_path, accepted_installation, changed): + plan, files, _, module, wheel = accepted_installation + path = files["result"][0] if changed == "result" else module if changed == "source" else wheel + path.write_bytes(path.read_bytes() + b" ") + with pytest.raises(ValueError): + comparison.accepted_preflights_passed(tmp_path, plan) + + +@pytest.mark.parametrize("name,field,value", [ + ("bootstrap", "verifier_preflight", {"status": "pending"}), + ("manifest", "commit", "another-revision"), + ("input_config", "install_only", False), +]) +def test_required_preflight_mode_and_revision_cannot_be_bypassed(tmp_path, accepted_installation, name, field, value): + plan, files, evidence, _, _ = accepted_installation + path, data = files[name] + data[field] = value + path.write_text(json.dumps(data)) + evidence[name + "_sha256"] = comparison.digest(path) + with pytest.raises(ValueError): + comparison.accepted_preflights_passed(tmp_path, plan) From 29cfddf06b2dca753e8f51d49016593df296d152 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Sun, 11 Oct 2026 10:23:40 -0700 Subject: [PATCH 5/5] test(benchmarks): record completed 12-task main baseline --- .../configs/dev12-baseline-20261011/README.md | 5 + .../reports/dev12-main-baseline-20261011.md | 220 ++ .../results/dev12-main-baseline-20261011.json | 1865 +++++++++++++++++ 3 files changed, 2090 insertions(+) create mode 100644 benchmarks/harbor/reports/dev12-main-baseline-20261011.md create mode 100644 benchmarks/harbor/results/dev12-main-baseline-20261011.json diff --git a/benchmarks/harbor/configs/dev12-baseline-20261011/README.md b/benchmarks/harbor/configs/dev12-baseline-20261011/README.md index c1ce6c3..d116afa 100644 --- a/benchmarks/harbor/configs/dev12-baseline-20261011/README.md +++ b/benchmarks/harbor/configs/dev12-baseline-20261011/README.md @@ -6,6 +6,11 @@ packages, reading their instructions, or seeing outcomes. It is a purposive **development** cohort, not a representative random sample of Terminal-Bench. The other 55 tasks are left unopened by this experiment. +Completed result: **10/12 passed**, with two scored correctness failures and no +final setup or verifier errors. See the +[baseline report](../../reports/dev12-main-baseline-20261011.md) and +[machine-readable evidence](../../results/dev12-main-baseline-20261011.json). + The runtime is main at `2eb99b608dfb39464d17d109a4a8f11afda607a9`, isolated in `jobs/dev12-baseline-main-source`. This branch records configuration and evidence; it does not change the agent or adapter. Neither prompt candidate from PR #48 diff --git a/benchmarks/harbor/reports/dev12-main-baseline-20261011.md b/benchmarks/harbor/reports/dev12-main-baseline-20261011.md new file mode 100644 index 0000000..9113970 --- /dev/null +++ b/benchmarks/harbor/reports/dev12-main-baseline-20261011.md @@ -0,0 +1,220 @@ +# Twelve-task main baseline, 2026-10-11 + +The frozen main runtime passed **10 of 12 tasks (83.3%)**. Both remaining +tasks reached the official assertions and failed on output correctness. There +were no final setup or verifier errors in these twelve scored trials. This is +a new development baseline, with one run per task; it is not a comparison with +the prompt candidates in PR #48. + +The failures expose two concrete verification gaps: losing original evidence +before recovery, and validating an implementation against a reference that +shares its unproven assumption. More self-tests alone would not address either. + +## Runtime and cohort + +- Runtime: main `2eb99b608dfb39464d17d109a4a8f11afda607a9`, in the detached + `jobs/dev12-baseline-main-source` worktree. Agent and adapter source are + unchanged. The reporting branch contains only benchmark infrastructure, + configurations, and evidence. +- Model: `deepseek/deepseek-flash` at + `https://api.deepseek.com/anthropic`; 100 replies, 65,536 output tokens per + request; temperature and reasoning effort unset. The original 385-word + headless prompt is frozen in the configuration directory. +- Dataset: `terminal-bench/terminal-bench-2-1`, revision + `sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a`. + Twelve tasks were selected by public names and coverage before their packages + were downloaded or their outcomes known. They extend the previously exercised + 22 tasks to a 34-task development pool. The other 55 packages remain unopened + by this experiment. This purposive sample is not a full-dataset score or an + estimate of repeated-run reliability. +- Each task ran once, in the registered order, with concurrency two and a + barrier between consecutive pairs. No task was replaced and no complete + model trial was retried. Work limits are native agent limits minus 180 seconds; + setup has a separate 1,800-second allowance. Original verifier limits remain + unchanged, with at most three attempts and only timeouts permitting a retry. + +[Selection and configuration](../configs/dev12-baseline-20261011/README.md), +[active plan](../configs/dev12-baseline-20261011/experiment-v3.json), and +[machine-readable results](../results/dev12-main-baseline-20261011.json) contain +the revisions, budgets, artifact hashes, metrics, and post-run diagnosis. + +## Results + +Agent time below is Harbor's agent-execution stage, excluding environment +startup, agent installation, and the official verifier. Test counts refer to +the official task suite; the security task also ran 367 project tests. + +| Task | Reward | Official tests | Agent seconds | Observed self-verification | +| --- | ---: | --- | ---: | --- | +| `cobol-modernization` | 1 | 3 passed | 505.2 | Executed original COBOL as an oracle; byte comparisons, exit behavior, 155 differential and 120 fuzz cases. | +| `fix-code-vulnerability` | 1 | 6 passed | 66.3 | Reproduced a failing header-validation test, then 367 project tests passed; exercised the WSGI entry point. | +| `nginx-request-logging` | 1 | 8 passed | 34.1 | Real HTTP requests, 404 content, log fields, and a concurrent rate-limit probe. | +| `git-multibranch` | 1 | 1 passed | 232.1 | Password SSH clone/push, both HTTPS deployment paths, deployment timing, and service restart. | +| `db-wal-recovery` | 0 | 5 passed, 2 failed | 302.0 | Checked structure and consistency of reconstructed data, without the lost original WAL as an oracle. | +| `multi-source-data-merger` | 1 | 3 passed | 55.8 | Synthetic mixed-alias case found a dropped-record bug; fixed it and checked final values and conflicts. | +| `query-optimize` | 1 | 6 passed | 536.7 | Full output comparison with original SQL; separate Python reference covered all 17,834 filtered rows. | +| `adaptive-rejection-sampler` | 1 | 9 passed | 566.0 | Distribution moments, quantiles, KS checks, invalid inputs, reproducibility, and saved-sample checks. | +| `pytorch-model-cli` | 0 | 5 passed, 1 failed | 298.7 | 330 image comparisons passed against a reference using the same inferred normalization. | +| `constraints-scheduling` | 1 | 3 passed | 30.2 | Minute-resolution search for the earliest valid slot, then calendar structure and conflict checks. | +| `custom-memory-heap-crash` | 1 | 6 passed | 280.7 | Reproduced release crash; debug/release runs, debugger allocation-site checks, and zero-error Valgrind runs. | +| `build-pov-ray` | 1 | 3 passed | 393.0 | Official-source build, deterministic renders, height-field regression, and reproducible rebuild. | + +These observations come from tool outputs, not just the agents' final claims. +Examples are ATIF steps 71/83/91 for COBOL, 15/19/31 for the security repair, +13/15/17/19 for data merging, 21/31/43 for SQL, and 59/61/65 for the C++ repair. +The sampler corrected a mistaken self-test expectation at step 20 after a +10/11 result; its subsequent saved samples were checked separately at step 22. +Passing the official suite does not establish correctness outside its coverage. + +## Failure: database recovery destroyed its own evidence + +The original task required recovering all eleven records, **including WAL +changes**, and testing their specific values. At ATIF step 5, the agent ran +`sqlite3 main.db` before copying the database and WAL, concurrently with a +640-byte WAL hexdump. At step 7, opening `main.db-wal` failed with +`FileNotFoundError`; step 9 confirmed it was gone. The trace supports SQLite's +open/close as the cause, which the agent also recognized at step 11. + +The agent identified XOR key `0x42`, but only the captured prefix survived. It +then guessed the six missing rows from the alphabetical fruit names and +`value = id * 100` pattern, wrote replacement database/WAL files, and exported +that reconstruction. The final message disclosed the loss and guessing. + +Step 55 verified JSON shape, eleven sorted rows, and agreement between SQLite +and JSON. Those checks compared two artifacts derived from the same guesses. +They could not establish recovery of the original values. The official verifier +found row 1 still at `100`, while its committed WAL update required `150`; +`test_recovered_data_completeness` and `test_wal_was_decrypted` failed. The +failure therefore concerns data provenance and preservation, not JSON syntax +or a verifier dependency problem. + +A useful agent change is to preserve complete original inputs before opening +stateful formats, perform recovery on copies, and distinguish recovered facts +from inferred replacements. Verification must check committed updates as well +as inserted row counts. It must not treat agreement between two guessed +artifacts as evidence that the original data was recovered. + +## Failure: model conversion used a shared, unsupported assumption + +The supplied `model.py` defines a 784–16–16–10 ReLU network. The agent extracted +its weights, then searched preprocessing choices against public MNIST labels. +At steps 51/53, `(pixel / 255 - 0.5) / 0.5` achieved 92.07% label accuracy, so +the agent chose it over plain `pixel / 255`. This changed the requested model's +observable predictions to improve a different objective: digit-label accuracy. + +At steps 75/77/83/87, the C++ CLI matched a separately written Python reference +on 330 images, including small-margin cases. Both implementations read the +same inferred normalization from the generated weights file. This was useful +for detecting implementation discrepancies, but did not validate the assumption +itself. The agent's final message acknowledged that preprocessing was inferred. +The task instructions did not explicitly specify normalization, so this was a +real uncertainty that should have remained visible in its verification claim. + +The official verifier checked fixed model predictions and found 14 mismatches +among the first 50 MNIST test images. After all twelve outputs were frozen, a +separate diagnostic loaded the original float32 weights and implemented the +supplied architecture with standard-library float64 arithmetic: + +| Preprocessing | Mismatches against official expectations, first 50 images | +| --- | ---: | +| `pixel / 255` | 0 | +| `(pixel / 255 - 0.1307) / 0.3081` | 10 | +| `(pixel / 255 - 0.5) / 0.5` | 14 | + +The final variant reproduces every reported mismatch index. This confirms +preprocessing as a sufficient explanation for the observed failure. The +diagnostic used the original task model and a public MNIST fixture; their hashes +and all predictions are recorded in the results JSON. It did not edit the +submission, change the official score, or constitute another model trial. + +For model conversion, the agent should preserve source behavior and validate +the full input-to-output contract. An independent numerical implementation is +not an independent oracle when both programs share an unverified preprocessing +choice. Label accuracy should not substitute for behavioral equivalence. + +## Installation and verification accounting + +The original install-only preflight had five successes, six image-pull/start +errors, and one APT setup timeout. The second had eleven successes and one +partial uv download. Both complete histories remain in the +[first](../configs/dev12-baseline-20261011/preflight-v1-evidence.json) and +[second](../configs/dev12-baseline-20261011/preflight-v2-evidence.json) evidence +files. These eight preflight failures are separate from the twelve scored +trials. + +Before any model ran, plan v3 accepted exactly one successful native +installation per task: eleven from the second preflight and POV-Ray's success +from the first. The reusable orchestrator verified their task revision, source +snapshot, installed wheel, bootstrap state, installation-only mode, and artifact +hashes. No successful result was synthesized. The runtime, prompt, tasks, +budgets, and queue were unchanged across plan amendments. + +The branch also adds a verified cache for original public Docker Hub images. +Six stalled image downloads were completed through digest-checked parallel +ranges and imported without rebuilding or modifying images. Task scripts, +package sources, and official tests were preserved. All twelve verifier +dependency profiles remain **`not_configured`**: successful agent installation +was never reported as successful verifier-dependency preflight. + +The PyTorch verifier timed out once after 900 seconds during dependency setup; +its unchanged second attempt completed and reached the failing assertion. Every +other verifier completed on its first attempt. Each retry retained the native +per-attempt limit and the same agent output. All twelve final official suites +ran; there are no unresolved verifier errors to exclude from the denominator. + +The scored phase ran from **15:50:51 to 17:11:31 UTC**, approximately **80.7 +minutes** elapsed, excluding the earlier preflights. Summed across concurrent +trials, agent execution was 55.0 minutes, agent installation 39.9 minutes, and +verification 34.7 minutes. These sums are not wall-clock duration. The calendar +task alone spent 28.4 minutes installing, followed by 30.2 seconds of agent work. + +Recorded token-estimated cost totals **$0.600974**, with incomplete accounting: +PyTorch step 89 had a recoverable `RemoteProtocolError` and no usage record. +The sum covers known usage only, not an exact bill or complete total. Recorded +usage is 9,362,395 input tokens (9,171,328 cached) and 407,188 output tokens; +missing usage is not imputed as zero. This in-agent transport retry is separate +from the policy forbidding retries of entire model trials. + +## Audit boundaries and next experiments + +All twelve outputs had frozen by the observed checkpoint at 17:09:54 UTC, +before failure-content auditing began. Their trajectory hashes still match. +Final checks also matched the registered plan, prompt, task files, input +configurations, complete source snapshots, wheel digests, original image +identities/registry digests, and previously recorded completed-result hashes. +Queue barriers and verifier retry limits were verified from native artifacts. +No containers from the twelve scored trials remained after completion. + +Trace review found no observed retrieval of external benchmark solutions or +successful access to hidden verifier contents. POV-Ray used official upstream +archives; PyTorch used public MNIST images and labels, including examples later +used by the verifier. That overlap is recorded, not treated as benchmark-answer +retrieval. Several agents searched local files for references, and the Git and +database agents read bootstrap logs. These observations cannot establish that +the model had never encountered the benchmark in training. + +The database agent also read process environments at step 35, placing the +provider credential in its local trace. Raw logs must remain private or be +redacted before sharing. This report and the published JSON contain derived +observations and hashes, not raw environment values. Preventing credential +values from reaching tool observations is a separate agent-hardening follow-up. + +For a subsequent optimization experiment: + +1. Keep these twelve tasks intact. Use the two failures to develop preservation + and oracle-selection behavior; keep the ten passes as regression cases. +2. Require evidence for the decisive requirement: original data values for + recovery, source predictions for conversion. Track whether the oracle shares + assumptions or generated data with the implementation. +3. Add controlled development variants that change update values or input + contracts. Preserve the official baseline and label variants as custom tests. +4. Compare baseline and candidate with paired repeats and the same budgets; + measure task score, regressions, unsupported completion claims, and cost. + This run alone cannot show an improvement or estimate variance. +5. Leave the remaining 55 tasks unopened until a candidate and a fresh + evaluation protocol are fixed. Do not keep selecting fresh tasks after + seeing each development result. + +Infrastructure validation on this branch: **128 Harbor tests passed**, including +image-cache and accepted-preflight provenance tests. No agent optimization was +included in this baseline. diff --git a/benchmarks/harbor/results/dev12-main-baseline-20261011.json b/benchmarks/harbor/results/dev12-main-baseline-20261011.json new file mode 100644 index 0000000..0ce76fd --- /dev/null +++ b/benchmarks/harbor/results/dev12-main-baseline-20261011.json @@ -0,0 +1,1865 @@ +{ + "schema_version": 1, + "experiment": "dev12-main-baseline-20261011", + "collected_at": "2026-10-11T17:23:17.120884+00:00", + "cohort": "purposive_development", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "runtime_source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "plan_path": "benchmarks/harbor/configs/dev12-baseline-20261011/experiment-v3.json", + "plan_sha256": "9dcb64a3bbe76482dbb0aeb9297d2a1251071b704ad78e9f2b77ebc5040a2c9a", + "model_output_freeze": { + "observed_frozen_at": "2026-10-11T17:09:54.227372+00:00", + "model_outputs": 12, + "trials": [ + { + "task": "terminal-bench/cobol-modernization", + "trajectory_path": "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011/cobol-modernization__GQnVApW/agent/trajectory.json", + "trajectory_sha256": "4f8f83b3c39424085355f531f057d6479c145a52ed85f651be263ba292e7b04e", + "terminal": { + "status": "completed", + "duration_ms": 504192.364566, + "timestamp": "2026-10-11T15:59:56.042Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + } + }, + { + "task": "terminal-bench/fix-code-vulnerability", + "trajectory_path": "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011/fix-code-vulnerability__8PT7Mmq/agent/trajectory.json", + "trajectory_sha256": "7b9c7f7e56b664665dc86e4aeb960364b0548277a9b466b052683cb1cde49293", + "terminal": { + "status": "completed", + "duration_ms": 65406.544183, + "timestamp": "2026-10-11T15:52:31.589Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + } + }, + { + "task": "terminal-bench/nginx-request-logging", + "trajectory_path": "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011/nginx-request-logging__EabhPTP/agent/trajectory.json", + "trajectory_sha256": "eae4b902126963ee4cddcdb378587f1795c83ffb3a6ed798c3151f7290199778", + "terminal": { + "status": "completed", + "duration_ms": 33079.926325, + "timestamp": "2026-10-11T16:01:44.208Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + } + }, + { + "task": "terminal-bench/git-multibranch", + "trajectory_path": "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011/git-multibranch__69UpsQX/agent/trajectory.json", + "trajectory_sha256": "fddca6d5eb5b3bb341f539e1766a18e35367ab61082e198e740045a79a8e0300", + "terminal": { + "status": "completed", + "duration_ms": 231281.000847, + "timestamp": "2026-10-11T16:05:16.034Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + } + }, + { + "task": "terminal-bench/db-wal-recovery", + "trajectory_path": "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011/db-wal-recovery__vQr4RTJ/agent/trajectory.json", + "trajectory_sha256": "99a8f85ab7c65218f67edf4c7a6ba15dd2ba2eae1bca0d4abdc74dcb66b7f1f9", + "terminal": { + "status": "completed", + "duration_ms": 300714.826752, + "timestamp": "2026-10-11T16:13:14.583Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + } + }, + { + "task": "terminal-bench/multi-source-data-merger", + "trajectory_path": "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011/multi-source-data-merger__H5tGDAu/agent/trajectory.json", + "trajectory_sha256": "98254f523e1af69d46ce666756d5d0a0d735663a4a2651a3fa9534b70214dc2e", + "terminal": { + "status": "completed", + "duration_ms": 54941.240413, + "timestamp": "2026-10-11T16:08:57.994Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + } + }, + { + "task": "terminal-bench/query-optimize", + "trajectory_path": "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011/query-optimize__xW8CdxE/agent/trajectory.json", + "trajectory_sha256": "d1e4af3f2b506732e2a39901e6608c88697f3e21b0202f26638b346abd40de6d", + "terminal": { + "status": "completed", + "duration_ms": 535870.372246, + "timestamp": "2026-10-11T16:28:00.417Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + } + }, + { + "task": "terminal-bench/adaptive-rejection-sampler", + "trajectory_path": "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011/adaptive-rejection-sampler__AbftRyA/agent/trajectory.json", + "trajectory_sha256": "14c739071f532119612b23f97ceab1d79cdedd6e7f4eb23307b1627e38540019", + "terminal": { + "status": "completed", + "duration_ms": 565096.307358, + "timestamp": "2026-10-11T16:28:36.521Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + } + }, + { + "task": "terminal-bench/pytorch-model-cli", + "trajectory_path": "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011/pytorch-model-cli__asD2fTE/agent/trajectory.json", + "trajectory_sha256": "ac3a14da4dc07a62b48fb509f094003fc0913dfb1a05d65b3a44aa026e3e9bc4", + "terminal": { + "status": "completed", + "duration_ms": 297760.896708, + "timestamp": "2026-10-11T16:40:24.130Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + } + }, + { + "task": "terminal-bench/constraints-scheduling", + "trajectory_path": "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011/constraints-scheduling__NPnsKa5/agent/trajectory.json", + "trajectory_sha256": "2d1974f6890199437dc2a8719006f553907a8a9976c518a4a70b8aa13091b44d", + "terminal": { + "status": "completed", + "duration_ms": 29296.936721, + "timestamp": "2026-10-11T16:59:47.232Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + } + }, + { + "task": "terminal-bench/custom-memory-heap-crash", + "trajectory_path": "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011/custom-memory-heap-crash__sUefumj/agent/trajectory.json", + "trajectory_sha256": "633e9240d85490bfba3bbe2fdc074f3a860b5da1bdd6fa545e21b78b83ac5ad7", + "terminal": { + "status": "completed", + "duration_ms": 279820.513108, + "timestamp": "2026-10-11T17:05:50.190Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + } + }, + { + "task": "terminal-bench/build-pov-ray", + "trajectory_path": "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011/build-pov-ray__gSzbLYX/agent/trajectory.json", + "trajectory_sha256": "2a0e42851a8c3b0c8ee45f0dc3db1882eae3ba72cfd1fa883b181200d20588b3", + "terminal": { + "status": "completed", + "duration_ms": 392133.312638, + "timestamp": "2026-10-11T17:07:50.736Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + } + } + ] + }, + "integrity": { + "registration_unchanged": true, + "plan_source_prompt_configs_tasks_unchanged": true, + "accepted_preflights_verified": 12, + "source_snapshots_verified": 12, + "wheel_digests_verified": 12, + "trajectory_freeze_verified": 12, + "earlier_completed_artifact_hashes_verified": 8, + "all_native_test_suites_executed": true, + "queue_pair_barriers_verified": true, + "whole_model_trial_retries": 0, + "outcome_based_replacements": 0 + }, + "aggregate": { + "trials": 12, + "raw_passes": 10, + "scored_failures": 2, + "setup_errors": 0, + "verifier_errors": 0, + "unsupported_verifier_dependency_profiles": 12, + "pass_rate": 0.8333333333333334, + "verifier_timeout_retries": 1, + "estimated_known_cost_usd": 0.600973668, + "cost_complete": false, + "usage_incomplete_tasks": [ + "terminal-bench/pytorch-model-cli" + ], + "recorded_tokens": { + "prompt_tokens": 9362395, + "completion_tokens": 407188, + "cached_tokens": 9171328 + }, + "summed_stage_seconds": { + "environment_setup": 11.621372000000001, + "agent_setup": 2395.148702, + "agent_execution": 3300.751312, + "verifier": 2081.279393 + }, + "model_job_wall_seconds": 4839.714099 + }, + "trials": [ + { + "order": 1, + "task": "terminal-bench/cobol-modernization", + "task_ref": "sha256:e03aa03965eee48c9f772d566ff9bcd741fceeac9693123f82a13fdd46be0d88", + "trial": "cobol-modernization__GQnVApW", + "job": "dev12-cobol-modernization-baseline-20261011", + "raw_reward": 1.0, + "classification": "scored_pass", + "exception": null, + "official_test_summary": { + "tests": 3, + "passed": 3, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1791734414.6418216, + "stop": 1791734414.689376 + }, + "work_budget_seconds": 720, + "started_at": "2026-10-11T15:50:51.866843Z", + "finished_at": "2026-10-11T16:00:25.509548Z", + "seconds": { + "environment_setup": 0.933953, + "agent_setup": 38.24792, + "agent_execution": 505.160568, + "verifier": 18.268325 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 18.05617724900003 + } + ], + "verifier_timeout_seconds": 900.0, + "verifier_dependency_profile": "not_configured", + "recorded_agent_steps": 47, + "incomplete_model_calls": [], + "tool_calls": 47, + "accounting": { + "usage_complete": true, + "recorded_tokens": { + "prompt_tokens": 2698556, + "completion_tokens": 82809, + "cached_tokens": 2672896 + }, + "estimated_known_cost_usd": 0.123106176, + "final_metrics": { + "total_steps": 95, + "total_prompt_tokens": 2698556, + "total_completion_tokens": 82809, + "total_cached_tokens": 2672896, + "total_cost_usd": 0.123106176, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.123106176 + } + } + }, + "task_checksum": "5e3806427eaaaa9f0f756a2a87d4f7f50ec1c8a0d329b26bab7dc09657b937cf", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011/cobol-modernization__GQnVApW/result.json": "ef3d278f8f7fb0a7af1b633dcf7266339016a620fcb7d77d38246e4cfcb73fc1", + "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011/cobol-modernization__GQnVApW/agent/trajectory.json": "4f8f83b3c39424085355f531f057d6479c145a52ed85f651be263ba292e7b04e", + "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011/cobol-modernization__GQnVApW/agent/bootstrap.json": "1812b1d7452b4ec930ab65be85cd8ba7f2e96342153c5e9e7d0dc948f342502f", + "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011/cobol-modernization__GQnVApW/verifier/retry-summary.json": "66e9157d51143a3d288ff4bcde44b055699fd01684ed07a7d2988b80af5fa076", + "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011/cobol-modernization__GQnVApW/verifier/test-stdout.txt": "f10b9a00a36e6d2fbf8bc3951c0b1630b91de506bdf0231a64ec95c6771cad1d", + "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011/cobol-modernization__GQnVApW/verifier/ctrf.json": "ae2aeb35fa7b1952ed2f64ff83cb1aa2341d1a3cb5a49b0d74c57bb56b33477e", + "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011/cobol-modernization__GQnVApW/verifier/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011-input/manifest.json": "a9ec1f89510ffacff9bef73cbf4decd7352edbd2bc54d2911382c1bdf5841b09", + "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011-input/config.json": "43e5dc00efba212c737ae12ccec37fd6e2e7da355f8b3b5f35f768089771d653", + "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev12-baseline-20261011-v3-orchestration/dev12-cobol-modernization-baseline-20261011.json": "94101e9a1f8045ea8791596acea9f2c8d993d889d7f61f269633350a77d9dcd7", + "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011/cobol-modernization__GQnVApW/verifier/attempts/1/ctrf.json": "ae2aeb35fa7b1952ed2f64ff83cb1aa2341d1a3cb5a49b0d74c57bb56b33477e", + "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011/cobol-modernization__GQnVApW/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-cobol-modernization-baseline-20261011/cobol-modernization__GQnVApW/verifier/attempts/1/test-stdout.txt": "f10b9a00a36e6d2fbf8bc3951c0b1630b91de506bdf0231a64ec95c6771cad1d" + } + }, + { + "order": 2, + "task": "terminal-bench/fix-code-vulnerability", + "task_ref": "sha256:d31348aa16b533a15f013420e4d9726dc529e6086adc7d6d48067d22cc18fe71", + "trial": "fix-code-vulnerability__8PT7Mmq", + "job": "dev12-fix-code-vulnerability-baseline-20261011", + "raw_reward": 1.0, + "classification": "scored_pass", + "exception": null, + "official_test_summary": { + "tests": 6, + "passed": 6, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1791733961.4188828, + "stop": 1791733961.4372432 + }, + "work_budget_seconds": 720, + "started_at": "2026-10-11T15:50:51.470967Z", + "finished_at": "2026-10-11T15:52:52.233260Z", + "seconds": { + "environment_setup": 0.9524, + "agent_setup": 33.038588, + "agent_execution": 66.275879, + "verifier": 9.522895 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 9.333147841000027 + } + ], + "verifier_timeout_seconds": 900.0, + "verifier_dependency_profile": "not_configured", + "recorded_agent_steps": 16, + "incomplete_model_calls": [], + "tool_calls": 21, + "accounting": { + "usage_complete": true, + "recorded_tokens": { + "prompt_tokens": 162083, + "completion_tokens": 7594, + "cached_tokens": 151424 + }, + "estimated_known_cost_usd": 0.013219044, + "final_metrics": { + "total_steps": 33, + "total_prompt_tokens": 162083, + "total_completion_tokens": 7594, + "total_cached_tokens": 151424, + "total_cost_usd": 0.013219044, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.013219044 + } + } + }, + "task_checksum": "80d707a96acfd0e19af96f0f0c8f44deefdbf167538fefbc0d1dc6391392090a", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011/fix-code-vulnerability__8PT7Mmq/result.json": "53b9cf6b726ac1c7131ad7eabef0f4ce9e8e7c54134206ab0b02b29e69b39d2d", + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011/fix-code-vulnerability__8PT7Mmq/agent/trajectory.json": "7b9c7f7e56b664665dc86e4aeb960364b0548277a9b466b052683cb1cde49293", + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011/fix-code-vulnerability__8PT7Mmq/agent/bootstrap.json": "8d1fa3e359c1a2c87bb9b262c9113a423af3445aa5a20da8a2e6fb4d63b4f67a", + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011/fix-code-vulnerability__8PT7Mmq/verifier/retry-summary.json": "6da7d47a863e50dd5b2e910823ddbb81a6b2910ec0fc80c308d54c3bf532c4e3", + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011/fix-code-vulnerability__8PT7Mmq/verifier/test-stdout.txt": "5b1c1b620157e4e7ee7e5368a1cd9f8de20235ae042eaeeb7f266332d7e38f69", + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011/fix-code-vulnerability__8PT7Mmq/verifier/ctrf.json": "999592c0d6e4cfa5516e44db1e208a3f3df9f0f50356a9dd9968fdd1598e801d", + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011/fix-code-vulnerability__8PT7Mmq/verifier/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011-input/manifest.json": "20b441c2dfc5a7c9ece86670859a49d8f2db355165c400b691d45d69e539babd", + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011-input/config.json": "1f6958d0ca945eca970719d64a87a2dc5994bc0254cd71937060689c3863d5de", + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev12-baseline-20261011-v3-orchestration/dev12-fix-code-vulnerability-baseline-20261011.json": "c2f0153ace0b2eea1fc2597055651f836dd1c7b60f2ddd2ff3c14b903022cb0b", + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011/fix-code-vulnerability__8PT7Mmq/verifier/attempts/1/ctrf.json": "999592c0d6e4cfa5516e44db1e208a3f3df9f0f50356a9dd9968fdd1598e801d", + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011/fix-code-vulnerability__8PT7Mmq/verifier/attempts/1/original-repo-ctrf.json": "f4fc9648eec7ccf9a0891f567624303f7c6c6636b1cbec27917c92e8d3068eed", + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011/fix-code-vulnerability__8PT7Mmq/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-fix-code-vulnerability-baseline-20261011/fix-code-vulnerability__8PT7Mmq/verifier/attempts/1/test-stdout.txt": "5b1c1b620157e4e7ee7e5368a1cd9f8de20235ae042eaeeb7f266332d7e38f69" + } + }, + { + "order": 3, + "task": "terminal-bench/nginx-request-logging", + "task_ref": "sha256:9d1b8bebd989ea0bc8080c3b159480068caf183c5fd61385868b2574b206e097", + "trial": "nginx-request-logging__EabhPTP", + "job": "dev12-nginx-request-logging-baseline-20261011", + "raw_reward": 1.0, + "classification": "scored_pass", + "exception": null, + "official_test_summary": { + "tests": 8, + "passed": 8, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1791734525.1599152, + "stop": 1791734527.3533282 + }, + "work_budget_seconds": 720, + "started_at": "2026-10-11T16:00:31.803481Z", + "finished_at": "2026-10-11T16:02:18.275923Z", + "seconds": { + "environment_setup": 1.033164, + "agent_setup": 37.474728, + "agent_execution": 34.05106, + "verifier": 22.80059 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 22.611146957999154 + } + ], + "verifier_timeout_seconds": 900.0, + "verifier_dependency_profile": "not_configured", + "recorded_agent_steps": 11, + "incomplete_model_calls": [], + "tool_calls": 14, + "accounting": { + "usage_complete": true, + "recorded_tokens": { + "prompt_tokens": 57586, + "completion_tokens": 3408, + "cached_tokens": 53120 + }, + "estimated_known_cost_usd": 0.00574812, + "final_metrics": { + "total_steps": 23, + "total_prompt_tokens": 57586, + "total_completion_tokens": 3408, + "total_cached_tokens": 53120, + "total_cost_usd": 0.00574812, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.00574812 + } + } + }, + "task_checksum": "913305d8f286ff121b30e4893217142190b27f12952343f22c0999dd2a4d725e", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011/nginx-request-logging__EabhPTP/result.json": "9a431a143f83c13c17ad3c67d2bafaab9f7f90e7d0c7e3dd1b535ca28ee8c2b1", + "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011/nginx-request-logging__EabhPTP/agent/trajectory.json": "eae4b902126963ee4cddcdb378587f1795c83ffb3a6ed798c3151f7290199778", + "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011/nginx-request-logging__EabhPTP/agent/bootstrap.json": "bd617a3903bf50c818febe3abf77cc268952deb3cd9f0c6db99bc0cbf8146a94", + "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011/nginx-request-logging__EabhPTP/verifier/retry-summary.json": "99d7c7ccea0083dd20dcfbfda613b2501c9bd7f9ef5553862100d9e8a8cf6159", + "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011/nginx-request-logging__EabhPTP/verifier/test-stdout.txt": "eb7475abfeecda63ffb1c40f78f99d8cfcba9aea5ca883f5f1fb8cdc63f6f839", + "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011/nginx-request-logging__EabhPTP/verifier/ctrf.json": "b9bc0576b31ed205d3ea2fefd33d13542f23ad074d3972972d40a38f71af897d", + "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011/nginx-request-logging__EabhPTP/verifier/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011-input/manifest.json": "ae8579625d2ceaf69be0f948815a88b2a71fd5c578d2344705c2c2a333aa92a6", + "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011-input/config.json": "262ad1af0bc0cc6271a0c66ec6cc3c206fbdbb6aca6bcfdb47556e018131b3d0", + "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev12-baseline-20261011-v3-orchestration/dev12-nginx-request-logging-baseline-20261011.json": "83cf7476987ec4d90ef54bffd87f75579128f1dfc0f0b2b7b45067eedae351dc", + "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011/nginx-request-logging__EabhPTP/verifier/attempts/1/ctrf.json": "b9bc0576b31ed205d3ea2fefd33d13542f23ad074d3972972d40a38f71af897d", + "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011/nginx-request-logging__EabhPTP/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-nginx-request-logging-baseline-20261011/nginx-request-logging__EabhPTP/verifier/attempts/1/test-stdout.txt": "eb7475abfeecda63ffb1c40f78f99d8cfcba9aea5ca883f5f1fb8cdc63f6f839" + } + }, + { + "order": 4, + "task": "terminal-bench/git-multibranch", + "task_ref": "sha256:7759a92a373c967dd31535975bad8659410644025673fe62385444fd2605c5fa", + "trial": "git-multibranch__69UpsQX", + "job": "dev12-git-multibranch-baseline-20261011", + "raw_reward": 1.0, + "classification": "scored_pass", + "exception": null, + "official_test_summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1791734736.7768967, + "stop": 1791734827.94831 + }, + "work_budget_seconds": 720, + "started_at": "2026-10-11T16:00:31.786222Z", + "finished_at": "2026-10-11T16:07:18.794990Z", + "seconds": { + "environment_setup": 1.010187, + "agent_setup": 51.263702, + "agent_execution": 232.131266, + "verifier": 111.543161 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 111.34293115500077 + } + ], + "verifier_timeout_seconds": 900.0, + "verifier_dependency_profile": "not_configured", + "recorded_agent_steps": 22, + "incomplete_model_calls": [], + "tool_calls": 28, + "accounting": { + "usage_complete": true, + "recorded_tokens": { + "prompt_tokens": 330601, + "completion_tokens": 15423, + "cached_tokens": 319232 + }, + "estimated_known_cost_usd": 0.023833692, + "final_metrics": { + "total_steps": 45, + "total_prompt_tokens": 330601, + "total_completion_tokens": 15423, + "total_cached_tokens": 319232, + "total_cost_usd": 0.023833692, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.023833692 + } + } + }, + "task_checksum": "abe8a7f8f9dcb0e2170740e4735559b368cecd925ddee09c9ed81a7c67dce90a", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011/git-multibranch__69UpsQX/result.json": "cb58d7f495ff58ee47582d995d8dff1c49128504f95a158989b5a6df841a06e2", + "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011/git-multibranch__69UpsQX/agent/trajectory.json": "fddca6d5eb5b3bb341f539e1766a18e35367ab61082e198e740045a79a8e0300", + "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011/git-multibranch__69UpsQX/agent/bootstrap.json": "391e000959d10d8aa551310ebe7bddd0c70185a684ee8cb543b7e8bf57e5e63b", + "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011/git-multibranch__69UpsQX/verifier/retry-summary.json": "a1f62e3c9cb0d3870e9180b7d130f06ddfba26d49bbec118dd3dff07eec117f4", + "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011/git-multibranch__69UpsQX/verifier/test-stdout.txt": "86aaa4516464d803d36759ab0ad510b1ae1f46dfc345dfacb929e60b2a014748", + "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011/git-multibranch__69UpsQX/verifier/ctrf.json": "280c06fffd6f521a3e13bcbbd8f339f858d19b6e212f44d6ae25ee6a206f1c51", + "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011/git-multibranch__69UpsQX/verifier/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011-input/manifest.json": "97d82709aadd19c986a324d22b13608104a091cd70f3a85e55aef0c00590db11", + "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011-input/config.json": "9c234f1e18fb14666fa52447f5b81266a57d88184b411a5dda7cc05c3b3212a5", + "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev12-baseline-20261011-v3-orchestration/dev12-git-multibranch-baseline-20261011.json": "b827bcbc0fad918f546cf7f52df2abcedc0649473493b4275fc377ec9b68d796", + "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011/git-multibranch__69UpsQX/verifier/attempts/1/ctrf.json": "280c06fffd6f521a3e13bcbbd8f339f858d19b6e212f44d6ae25ee6a206f1c51", + "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011/git-multibranch__69UpsQX/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-git-multibranch-baseline-20261011/git-multibranch__69UpsQX/verifier/attempts/1/test-stdout.txt": "86aaa4516464d803d36759ab0ad510b1ae1f46dfc345dfacb929e60b2a014748" + } + }, + { + "order": 5, + "task": "terminal-bench/db-wal-recovery", + "task_ref": "sha256:07e9193513bfce038b9f9d84962ecc85683eee0a642d14ee6fb7104b086ddab4", + "trial": "db-wal-recovery__vQr4RTJ", + "job": "dev12-db-wal-recovery-baseline-20261011", + "raw_reward": 0.0, + "classification": "scored_failure", + "exception": null, + "official_test_summary": { + "tests": 7, + "passed": 5, + "failed": 2, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1791735481.9039369, + "stop": 1791735481.930828 + }, + "work_budget_seconds": 720, + "started_at": "2026-10-11T16:07:27.182279Z", + "finished_at": "2026-10-11T16:18:12.844155Z", + "seconds": { + "environment_setup": 0.925319, + "agent_setup": 44.577398, + "agent_execution": 302.041538, + "verifier": 287.027836 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 286.8307059710005 + } + ], + "verifier_timeout_seconds": 900.0, + "verifier_dependency_profile": "not_configured", + "recorded_agent_steps": 28, + "incomplete_model_calls": [], + "tool_calls": 37, + "accounting": { + "usage_complete": true, + "recorded_tokens": { + "prompt_tokens": 980554, + "completion_tokens": 54949, + "cached_tokens": 954624 + }, + "estimated_known_cost_usd": 0.079445544, + "final_metrics": { + "total_steps": 57, + "total_prompt_tokens": 980554, + "total_completion_tokens": 54949, + "total_cached_tokens": 954624, + "total_cost_usd": 0.079445544, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.079445544 + } + } + }, + "task_checksum": "c18abdc4fdc3a01bf374c55a9700708fe6a9662077d29db81abb692f0a3c5f6f", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011/db-wal-recovery__vQr4RTJ/result.json": "8554a35eb18153678c22b6a66a80a209a8b29e515d5e5765d6c46496b4c7b64c", + "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011/db-wal-recovery__vQr4RTJ/agent/trajectory.json": "99a8f85ab7c65218f67edf4c7a6ba15dd2ba2eae1bca0d4abdc74dcb66b7f1f9", + "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011/db-wal-recovery__vQr4RTJ/agent/bootstrap.json": "2ffe3efbf9fd9f423c3fc82102152f59af7f9180f8aaf39c46e61c96e2f02d76", + "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011/db-wal-recovery__vQr4RTJ/verifier/retry-summary.json": "e7e88e91a81f764ee441faf6f8a45a1f0ad31e1e4eb03df465d57bc16955ec43", + "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011/db-wal-recovery__vQr4RTJ/verifier/test-stdout.txt": "64d88e5e6fef9ba8484dbbde478a1f1e9fd5a4218a44473e5fa34eb9aa2d94d6", + "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011/db-wal-recovery__vQr4RTJ/verifier/ctrf.json": "68352fe9414a7e314215756dd4c5f2cc35a69fd783f34c698aa0d90e6942a3cd", + "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011/db-wal-recovery__vQr4RTJ/verifier/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011-input/manifest.json": "48cf72c126f4bb349feb89bcf8e1c843b97d56d2295eb04300bd84002f7f86b8", + "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011-input/config.json": "b450c1e0756f85f1e56c7434f4fee72483c50fd26ccab908a0da7e1c3dbc3ef3", + "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev12-baseline-20261011-v3-orchestration/dev12-db-wal-recovery-baseline-20261011.json": "939be6d5761096e1df9ab3f83b0b29e30beae7bdfbbf30252c24d0889febbe3d", + "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011/db-wal-recovery__vQr4RTJ/verifier/attempts/1/ctrf.json": "68352fe9414a7e314215756dd4c5f2cc35a69fd783f34c698aa0d90e6942a3cd", + "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011/db-wal-recovery__vQr4RTJ/verifier/attempts/1/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "jobs/dev12-baseline-main-source/jobs/dev12-db-wal-recovery-baseline-20261011/db-wal-recovery__vQr4RTJ/verifier/attempts/1/test-stdout.txt": "64d88e5e6fef9ba8484dbbde478a1f1e9fd5a4218a44473e5fa34eb9aa2d94d6" + } + }, + { + "order": 6, + "task": "terminal-bench/multi-source-data-merger", + "task_ref": "sha256:70367c38732e1beda7b229968a48d60242277c2fa4db91339c3f064c4c230d49", + "trial": "multi-source-data-merger__H5tGDAu", + "job": "dev12-multi-source-data-merger-baseline-20261011", + "raw_reward": 1.0, + "classification": "scored_pass", + "exception": null, + "official_test_summary": { + "tests": 3, + "passed": 3, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1791735045.8595047, + "stop": 1791735046.8771257 + }, + "work_budget_seconds": 720, + "started_at": "2026-10-11T16:07:26.456836Z", + "finished_at": "2026-10-11T16:10:58.054981Z", + "seconds": { + "environment_setup": 0.968286, + "agent_setup": 34.912998, + "agent_execution": 55.807942, + "verifier": 108.678375 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 108.00855693199992 + } + ], + "verifier_timeout_seconds": 900.0, + "verifier_dependency_profile": "not_configured", + "recorded_agent_steps": 10, + "incomplete_model_calls": [], + "tool_calls": 10, + "accounting": { + "usage_complete": true, + "recorded_tokens": { + "prompt_tokens": 88329, + "completion_tokens": 9546, + "cached_tokens": 83456 + }, + "estimated_known_cost_usd": 0.013417836, + "final_metrics": { + "total_steps": 21, + "total_prompt_tokens": 88329, + "total_completion_tokens": 9546, + "total_cached_tokens": 83456, + "total_cost_usd": 0.013417836, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.013417836 + } + } + }, + "task_checksum": "33fa3b988ff60ec62b6ce40ee455208cb083ac1ac46ddc5247e954c88b9d5e8e", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011/multi-source-data-merger__H5tGDAu/result.json": "bf39627d51baef3ad24da8b801916d6612c2a5c59122da0394a8dc53a39cf042", + "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011/multi-source-data-merger__H5tGDAu/agent/trajectory.json": "98254f523e1af69d46ce666756d5d0a0d735663a4a2651a3fa9534b70214dc2e", + "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011/multi-source-data-merger__H5tGDAu/agent/bootstrap.json": "43369761e1c62328eb0c3890c99d441d724fe366a178abf9c568928eb85c4174", + "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011/multi-source-data-merger__H5tGDAu/verifier/retry-summary.json": "9cba27413140de8aab08f28c7cdd299f371e776ee35f976ca1b6427e3c5bf53f", + "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011/multi-source-data-merger__H5tGDAu/verifier/test-stdout.txt": "59e3a95584098847fca0c5263180174b6216f17028c800710c7e32198d1eec1a", + "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011/multi-source-data-merger__H5tGDAu/verifier/ctrf.json": "694f1880784dfaa07c1c2f3d3869ded9385422222de599a1daec8a844ecc838c", + "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011/multi-source-data-merger__H5tGDAu/verifier/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011-input/manifest.json": "054ba4498970383fe0ff6e14d30d04354b0126cce80ba5da094da500bc3e2793", + "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011-input/config.json": "caf972eaef443375ef4e532fd413bf6a871e98a0f0c2561a07b3b6d5470091d6", + "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev12-baseline-20261011-v3-orchestration/dev12-multi-source-data-merger-baseline-20261011.json": "c535138aa69a4a2afd0f0d09ddaaf8e93f19e00c8693992afbdf030f8b90b162", + "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011/multi-source-data-merger__H5tGDAu/verifier/attempts/1/ctrf.json": "694f1880784dfaa07c1c2f3d3869ded9385422222de599a1daec8a844ecc838c", + "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011/multi-source-data-merger__H5tGDAu/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-multi-source-data-merger-baseline-20261011/multi-source-data-merger__H5tGDAu/verifier/attempts/1/test-stdout.txt": "59e3a95584098847fca0c5263180174b6216f17028c800710c7e32198d1eec1a" + } + }, + { + "order": 7, + "task": "terminal-bench/query-optimize", + "task_ref": "sha256:169496ea6843cb403b0860132675baf7aa0df0ac8b86221068e27d86395260f4", + "trial": "query-optimize__xW8CdxE", + "job": "dev12-query-optimize-baseline-20261011", + "raw_reward": 1.0, + "classification": "scored_pass", + "exception": null, + "official_test_summary": { + "tests": 6, + "passed": 6, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1791736097.5172405, + "stop": 1791736228.6854308 + }, + "work_budget_seconds": 720, + "started_at": "2026-10-11T16:18:20.667016Z", + "finished_at": "2026-10-11T16:30:39.573437Z", + "seconds": { + "environment_setup": 0.965353, + "agent_setup": 42.219075, + "agent_execution": 536.72064, + "verifier": 147.905097 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 147.62170704900018 + } + ], + "verifier_timeout_seconds": 1800.0, + "verifier_dependency_profile": "not_configured", + "recorded_agent_steps": 22, + "incomplete_model_calls": [], + "tool_calls": 29, + "accounting": { + "usage_complete": true, + "recorded_tokens": { + "prompt_tokens": 479388, + "completion_tokens": 24381, + "cached_tokens": 459008 + }, + "estimated_known_cost_usd": 0.038125248, + "final_metrics": { + "total_steps": 45, + "total_prompt_tokens": 479388, + "total_completion_tokens": 24381, + "total_cached_tokens": 459008, + "total_cost_usd": 0.038125248, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.038125248 + } + } + }, + "task_checksum": "593405b6a2f6970f0f4c2ca2aff455a3b32bce3eecf8b506cfeeb0fe93b14838", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011/query-optimize__xW8CdxE/result.json": "a22011a054a45ffb2d3ee876630e77e83718bb017edb98415b4c040dbf21208a", + "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011/query-optimize__xW8CdxE/agent/trajectory.json": "d1e4af3f2b506732e2a39901e6608c88697f3e21b0202f26638b346abd40de6d", + "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011/query-optimize__xW8CdxE/agent/bootstrap.json": "0a52c1c56dbec333ab809d00b69469021e06f49e0570249b280b6aacef2587f0", + "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011/query-optimize__xW8CdxE/verifier/retry-summary.json": "5e3b743fa2e363f6638a666bbd32d5fc8c1517443b0673b0d3ce5102cbcf7dc5", + "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011/query-optimize__xW8CdxE/verifier/test-stdout.txt": "ef549e4b73a37d7bfb4fbfb7b1a4dcf6790f97efb4f49b2e44eddab30fe0d583", + "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011/query-optimize__xW8CdxE/verifier/ctrf.json": "477608f55e8083337dda43de81168129271df20fe3e6ec36726c785933bf4feb", + "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011/query-optimize__xW8CdxE/verifier/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011-input/manifest.json": "30be5279642e73ce991102a668792c93e9a3185fb63579899106040fe53c5f56", + "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011-input/config.json": "172d1e2fb4b548aacab61cd1b74e46ac8df6d01d92bea3274a8ad7422b7585d6", + "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev12-baseline-20261011-v3-orchestration/dev12-query-optimize-baseline-20261011.json": "b2977822e159e02c4c71cf7a0bd18c8b2840632b854a6aeb3a072f467bf38d1b", + "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011/query-optimize__xW8CdxE/verifier/attempts/1/ctrf.json": "477608f55e8083337dda43de81168129271df20fe3e6ec36726c785933bf4feb", + "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011/query-optimize__xW8CdxE/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-query-optimize-baseline-20261011/query-optimize__xW8CdxE/verifier/attempts/1/test-stdout.txt": "ef549e4b73a37d7bfb4fbfb7b1a4dcf6790f97efb4f49b2e44eddab30fe0d583" + } + }, + { + "order": 8, + "task": "terminal-bench/adaptive-rejection-sampler", + "task_ref": "sha256:bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de", + "trial": "adaptive-rejection-sampler__AbftRyA", + "job": "dev12-adaptive-rejection-sampler-baseline-20261011", + "raw_reward": 1.0, + "classification": "scored_pass", + "exception": null, + "official_test_summary": { + "tests": 9, + "passed": 9, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1791736140.2109442, + "stop": 1791736144.8185015 + }, + "work_budget_seconds": 720, + "started_at": "2026-10-11T16:18:20.643245Z", + "finished_at": "2026-10-11T16:29:15.879694Z", + "seconds": { + "environment_setup": 0.996828, + "agent_setup": 49.072429, + "agent_execution": 565.986258, + "verifier": 27.949146 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 27.708376808999674 + } + ], + "verifier_timeout_seconds": 900.0, + "verifier_dependency_profile": "not_configured", + "recorded_agent_steps": 11, + "incomplete_model_calls": [], + "tool_calls": 11, + "accounting": { + "usage_complete": true, + "recorded_tokens": { + "prompt_tokens": 250525, + "completion_tokens": 102009, + "cached_tokens": 245632 + }, + "estimated_known_cost_usd": 0.125352492, + "final_metrics": { + "total_steps": 24, + "total_prompt_tokens": 250525, + "total_completion_tokens": 102009, + "total_cached_tokens": 245632, + "total_cost_usd": 0.125352492, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.125352492 + } + } + }, + "task_checksum": "a4726253ad23c3fc46b5cb9c3d21025874e367b18643a3920f636a1fa89e1207", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011/adaptive-rejection-sampler__AbftRyA/result.json": "bf64380d0c4c4cdaf6492c30dbb8ae31752a53e46d00d01433706dcf70ae079e", + "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011/adaptive-rejection-sampler__AbftRyA/agent/trajectory.json": "14c739071f532119612b23f97ceab1d79cdedd6e7f4eb23307b1627e38540019", + "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011/adaptive-rejection-sampler__AbftRyA/agent/bootstrap.json": "eb4895cdc7076ee2da4b9519eb1d623465f7c0828b9afb96a5d685f89a8c1f1b", + "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011/adaptive-rejection-sampler__AbftRyA/verifier/retry-summary.json": "3b768b047c61501aebbd6a5158cc0a79708b46123cfb79840c6c6ccf2ffd7630", + "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011/adaptive-rejection-sampler__AbftRyA/verifier/test-stdout.txt": "97f76fc515fef3921a46ec13cb78f2cbf66ec54dbd3540fe1373ba4bd67bc0ec", + "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011/adaptive-rejection-sampler__AbftRyA/verifier/ctrf.json": "9ef2a1b7e71afe901b69a01f156665f5d42b46b5c59154ce64d6be20e93b8456", + "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011/adaptive-rejection-sampler__AbftRyA/verifier/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011-input/manifest.json": "de657438e6731aae6e4f50708044294a344cc40245c93bb9a16d468aa1587c01", + "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011-input/config.json": "71ffc80b450b239ea585ee4d269ff6e77e47dd582506ec98162adc502f6e15b7", + "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev12-baseline-20261011-v3-orchestration/dev12-adaptive-rejection-sampler-baseline-20261011.json": "4b31628d579b050eaa9df0de522a82c7a4efdb3042e5634322d40e0d07e8f4a9", + "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011/adaptive-rejection-sampler__AbftRyA/verifier/attempts/1/ctrf.json": "9ef2a1b7e71afe901b69a01f156665f5d42b46b5c59154ce64d6be20e93b8456", + "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011/adaptive-rejection-sampler__AbftRyA/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-adaptive-rejection-sampler-baseline-20261011/adaptive-rejection-sampler__AbftRyA/verifier/attempts/1/test-stdout.txt": "97f76fc515fef3921a46ec13cb78f2cbf66ec54dbd3540fe1373ba4bd67bc0ec" + } + }, + { + "order": 9, + "task": "terminal-bench/pytorch-model-cli", + "task_ref": "sha256:6f11544dcc81a380dc4611902961f5520259c462d9fceb413419d66ab9a96c79", + "trial": "pytorch-model-cli__asD2fTE", + "job": "dev12-pytorch-model-cli-baseline-20261011", + "raw_reward": 0.0, + "classification": "scored_failure", + "exception": null, + "official_test_summary": { + "tests": 6, + "passed": 5, + "failed": 1, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1791737768.6626732, + "stop": 1791737790.0538428 + }, + "work_budget_seconds": 720, + "started_at": "2026-10-11T16:30:56.828069Z", + "finished_at": "2026-10-11T16:56:42.340334Z", + "seconds": { + "environment_setup": 0.964518, + "agent_setup": 267.780007, + "agent_execution": 298.721879, + "verifier": 966.520263 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "timeout", + "duration_seconds": 900.0019214439999 + }, + { + "attempt": 2, + "status": "completed", + "return_code": 0, + "duration_seconds": 66.32247663299859 + } + ], + "verifier_timeout_seconds": 900.0, + "verifier_dependency_profile": "not_configured", + "recorded_agent_steps": 46, + "incomplete_model_calls": [ + { + "step_id": 89, + "model_call_id": "model-38d54f7b-2d63-4ee3-a21a-98060dc808d7", + "error_type": "RemoteProtocolError", + "message": "peer closed connection without sending complete message body (incomplete chunked read)", + "generation_id": null, + "duration_ms": 1107.911214, + "will_retry": true, + "retry_delay_seconds": 1.0, + "source_timestamp": "2026-10-11T16:40:12.982Z", + "incomplete": true, + "started_at": "2026-10-11T16:40:11.874Z", + "started_at_source": "source_timestamp", + "timestamp_source": "source_timestamp" + } + ], + "tool_calls": 48, + "accounting": { + "usage_complete": false, + "recorded_tokens": { + "prompt_tokens": 1405137, + "completion_tokens": 37894, + "cached_tokens": 1384704 + }, + "estimated_known_cost_usd": 0.059910924, + "final_metrics": { + "total_steps": 93, + "extra": { + "usage_complete": false, + "known_cost_usd": 0.059910924, + "cost_is_partial": true, + "cost_is_estimated": true, + "estimated_cost_usd": 0.059910924 + } + } + }, + "task_checksum": "e8438573f503c2d98ad71905862638bfa485470010d15d5b092821e0096b9daa", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011/pytorch-model-cli__asD2fTE/result.json": "1e8182f609eaa0711cf791e3d0974fe380d6effe11447f9263483291bcc1d00c", + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011/pytorch-model-cli__asD2fTE/agent/trajectory.json": "ac3a14da4dc07a62b48fb509f094003fc0913dfb1a05d65b3a44aa026e3e9bc4", + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011/pytorch-model-cli__asD2fTE/agent/bootstrap.json": "9abae99e4800f37de01cc8c2eb9d13d0437c60d0c1c0724f648d551c7f382b77", + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011/pytorch-model-cli__asD2fTE/verifier/retry-summary.json": "ac94dd7803a3444e1c1843a015e8a8bdb14cef580445144661fd28cbdaedc2bd", + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011/pytorch-model-cli__asD2fTE/verifier/test-stdout.txt": "13431458fea4c92034f42c1caf51f2b8a607e1b8dddcd2c232902dc035f234f6", + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011/pytorch-model-cli__asD2fTE/verifier/ctrf.json": "f5a8d18263e441da6ac66499220f9e14b954513c26ed1986ec43cf61bd98d299", + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011/pytorch-model-cli__asD2fTE/verifier/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011-input/manifest.json": "a76f53880d39c493ec5e968691aa10639ff6622c35f5fb08db0673fe32787ff9", + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011-input/config.json": "52fe8b50f8814caf106afc4c4e022363f39a64185f9c98171dfec1357c29f7d7", + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev12-baseline-20261011-v3-orchestration/dev12-pytorch-model-cli-baseline-20261011.json": "8cdb2774c45999f9285460ea463ca13840a1b66836593283d3d769676613fe6f", + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011/pytorch-model-cli__asD2fTE/verifier/attempts/1/test-stdout.txt": "ad3cc444bf23a016870f0b5d03f09e3e3c0a4010de2cb88f6583a88035501654", + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011/pytorch-model-cli__asD2fTE/verifier/attempts/2/ctrf.json": "f5a8d18263e441da6ac66499220f9e14b954513c26ed1986ec43cf61bd98d299", + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011/pytorch-model-cli__asD2fTE/verifier/attempts/2/reward.txt": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "jobs/dev12-baseline-main-source/jobs/dev12-pytorch-model-cli-baseline-20261011/pytorch-model-cli__asD2fTE/verifier/attempts/2/test-stdout.txt": "13431458fea4c92034f42c1caf51f2b8a607e1b8dddcd2c232902dc035f234f6" + } + }, + { + "order": 10, + "task": "terminal-bench/constraints-scheduling", + "task_ref": "sha256:91dd6f87e1ee508328d5aafbd99a6bd1ccc47bc22671b4aaa4060bda492deb53", + "trial": "constraints-scheduling__NPnsKa5", + "job": "dev12-constraints-scheduling-baseline-20261011", + "raw_reward": 1.0, + "classification": "scored_pass", + "exception": null, + "official_test_summary": { + "tests": 3, + "passed": 3, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1791738006.8966916, + "stop": 1791738007.2491832 + }, + "work_budget_seconds": 1020, + "started_at": "2026-10-11T16:30:54.073053Z", + "finished_at": "2026-10-11T17:00:18.177450Z", + "seconds": { + "environment_setup": 0.985738, + "agent_setup": 1702.177106, + "agent_execution": 30.152862, + "verifier": 19.670661 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 19.47129868399861 + } + ], + "verifier_timeout_seconds": 1200.0, + "verifier_dependency_profile": "not_configured", + "recorded_agent_steps": 6, + "incomplete_model_calls": [], + "tool_calls": 5, + "accounting": { + "usage_complete": true, + "recorded_tokens": { + "prompt_tokens": 41911, + "completion_tokens": 5424, + "cached_tokens": 38016 + }, + "estimated_known_cost_usd": 0.007905396, + "final_metrics": { + "total_steps": 13, + "total_prompt_tokens": 41911, + "total_completion_tokens": 5424, + "total_cached_tokens": 38016, + "total_cost_usd": 0.007905396, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.007905396 + } + } + }, + "task_checksum": "5b311f974587adf67804532222c59395f41a6c984b54a5e96a56dd071ee801dc", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011/constraints-scheduling__NPnsKa5/result.json": "387f595a2e3e2583fcdd129ee0e2c39fb00cf36a5952350020e631a90cbf68ab", + "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011/constraints-scheduling__NPnsKa5/agent/trajectory.json": "2d1974f6890199437dc2a8719006f553907a8a9976c518a4a70b8aa13091b44d", + "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011/constraints-scheduling__NPnsKa5/agent/bootstrap.json": "c4d5399fb99e22fdaccbafaeca054f4d615062b69ddc2389af0ed1c19ba30325", + "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011/constraints-scheduling__NPnsKa5/verifier/retry-summary.json": "9e231af8730c507475558b0227f79427c294137e58dc76be04e77a8bbb6e4608", + "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011/constraints-scheduling__NPnsKa5/verifier/test-stdout.txt": "662f91bddfdd0ab684fde0983e4a47fe75c64990c9680373dd34d0bcc53e7f31", + "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011/constraints-scheduling__NPnsKa5/verifier/ctrf.json": "a5eea20278c556046d582c0e11e7ba3b326be02dc074c858a95d4bd715fe6449", + "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011/constraints-scheduling__NPnsKa5/verifier/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011-input/manifest.json": "3c54335b46550168c314fee8df3dd6c9b395f8fd749aa85869c229aceca23efc", + "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011-input/config.json": "72539609a630a73c5a417ee4ffec5bb8ad123b8cce58f60dc4b8fd88c9f769c9", + "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev12-baseline-20261011-v3-orchestration/dev12-constraints-scheduling-baseline-20261011.json": "3d6b4a3170e9294d2e707e891f261d527ce9ba43d75717578c06e878123fe673", + "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011/constraints-scheduling__NPnsKa5/verifier/attempts/1/ctrf.json": "a5eea20278c556046d582c0e11e7ba3b326be02dc074c858a95d4bd715fe6449", + "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011/constraints-scheduling__NPnsKa5/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-constraints-scheduling-baseline-20261011/constraints-scheduling__NPnsKa5/verifier/attempts/1/test-stdout.txt": "662f91bddfdd0ab684fde0983e4a47fe75c64990c9680373dd34d0bcc53e7f31" + } + }, + { + "order": 11, + "task": "terminal-bench/custom-memory-heap-crash", + "task_ref": "sha256:61e021e0304c9818a9be2c9173c8a0938ce5d136e95c5422e099f1fdb2dc080e", + "trial": "custom-memory-heap-crash__sUefumj", + "job": "dev12-custom-memory-heap-crash-baseline-20261011", + "raw_reward": 1.0, + "classification": "scored_pass", + "exception": null, + "official_test_summary": { + "tests": 6, + "passed": 6, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1791738676.7126513, + "stop": 1791738680.335953 + }, + "work_budget_seconds": 1620, + "started_at": "2026-10-11T17:00:25.666527Z", + "finished_at": "2026-10-11T17:11:31.185066Z", + "seconds": { + "environment_setup": 0.932353, + "agent_setup": 43.067384, + "agent_execution": 280.667347, + "verifier": 329.799339 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 329.5842780359999 + } + ], + "verifier_timeout_seconds": 1800.0, + "verifier_dependency_profile": "not_configured", + "recorded_agent_steps": 33, + "incomplete_model_calls": [], + "tool_calls": 49, + "accounting": { + "usage_complete": true, + "recorded_tokens": { + "prompt_tokens": 1153770, + "completion_tokens": 33784, + "cached_tokens": 1128960 + }, + "estimated_known_cost_usd": 0.05475756, + "final_metrics": { + "total_steps": 67, + "total_prompt_tokens": 1153770, + "total_completion_tokens": 33784, + "total_cached_tokens": 1128960, + "total_cost_usd": 0.05475756, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.05475756 + } + } + }, + "task_checksum": "a0db695f342c1b72f1b3d0b6c234eb802e970c4758931355e4ceeea4c9e73c37", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011/custom-memory-heap-crash__sUefumj/result.json": "464c0fb5a1d54336d978e84bd182545767bbc95a04d00c024826badb0b62ae7e", + "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011/custom-memory-heap-crash__sUefumj/agent/trajectory.json": "633e9240d85490bfba3bbe2fdc074f3a860b5da1bdd6fa545e21b78b83ac5ad7", + "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011/custom-memory-heap-crash__sUefumj/agent/bootstrap.json": "cf6bb84c42557ba2a7b7e0ec653bbc1c7cf86c236d1b7e2a6852918dfb8eb7dc", + "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011/custom-memory-heap-crash__sUefumj/verifier/retry-summary.json": "c0e89aff5086b8fad0b1070b14253cf8510398bad2f85cc94a30269688a78e83", + "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011/custom-memory-heap-crash__sUefumj/verifier/test-stdout.txt": "14f6704998da0aa999e43e00070c9294226313885c5ea80a07a43ca2b998d79f", + "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011/custom-memory-heap-crash__sUefumj/verifier/ctrf.json": "f6a20756708b696647523c2aab2e1d11f092fe2ef4f353dbc322fd5a35eaf4be", + "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011/custom-memory-heap-crash__sUefumj/verifier/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011-input/manifest.json": "59d97783114ebe4fa9f673a83d494a11f954a7e2695afba00249d58a21cc2645", + "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011-input/config.json": "199419c73ec813a4aab83df11085e36b5d51cc1aac7c410901bec30650e90783", + "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev12-baseline-20261011-v3-orchestration/dev12-custom-memory-heap-crash-baseline-20261011.json": "998ab1c22ebd6015842c0f9de6181544a9b61c00077c07d1709e1f124b79a7ef", + "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011/custom-memory-heap-crash__sUefumj/verifier/attempts/1/ctrf.json": "f6a20756708b696647523c2aab2e1d11f092fe2ef4f353dbc322fd5a35eaf4be", + "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011/custom-memory-heap-crash__sUefumj/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-custom-memory-heap-crash-baseline-20261011/custom-memory-heap-crash__sUefumj/verifier/attempts/1/test-stdout.txt": "14f6704998da0aa999e43e00070c9294226313885c5ea80a07a43ca2b998d79f" + } + }, + { + "order": 12, + "task": "terminal-bench/build-pov-ray", + "task_ref": "sha256:4f08dab602fe2e9e2f42051cc6aef684a88cf2822e8b499b3ca19e670759fddb", + "trial": "build-pov-ray__gSzbLYX", + "job": "dev12-build-pov-ray-baseline-20261011", + "raw_reward": 1.0, + "classification": "scored_pass", + "exception": null, + "official_test_summary": { + "tests": 3, + "passed": 3, + "failed": 0, + "skipped": 0, + "pending": 0, + "other": 0, + "start": 1791738494.844755, + "stop": 1791738502.6303985 + }, + "work_budget_seconds": 11820, + "started_at": "2026-10-11T17:00:25.604761Z", + "finished_at": "2026-10-11T17:08:33.715498Z", + "seconds": { + "environment_setup": 0.953273, + "agent_setup": 51.317367, + "agent_execution": 393.034073, + "verifier": 31.593705 + }, + "verifier_attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 31.405323345999932 + } + ], + "verifier_timeout_seconds": 12000.0, + "verifier_dependency_profile": "not_configured", + "recorded_agent_steps": 53, + "incomplete_model_calls": [], + "tool_calls": 59, + "accounting": { + "usage_complete": true, + "recorded_tokens": { + "prompt_tokens": 1713955, + "completion_tokens": 29967, + "cached_tokens": 1680256 + }, + "estimated_known_cost_usd": 0.056151636, + "final_metrics": { + "total_steps": 107, + "total_prompt_tokens": 1713955, + "total_completion_tokens": 29967, + "total_cached_tokens": 1680256, + "total_cost_usd": 0.056151636, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.056151636 + } + } + }, + "task_checksum": "3426fb4714b5bb0463cc30bd872712a2400b23934b135c40ef7f88b27da4cd7a", + "source_ref": "2eb99b608dfb39464d17d109a4a8f11afda607a9", + "source_sha256": { + "nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "nanopycodeagent/agent.py": "aa58960803be77797c2addf65d7cc785a4be54f40ad58d53b9eea9fca1cf4b56", + "nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "wheel_sha256": "f4dcc18758e584696f955ae57fd18c751599b70e8b497ca2db7ef9e2b16e9350", + "provenance_checks_passed": true, + "evidence_sha256": { + "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011/build-pov-ray__gSzbLYX/result.json": "1b9b1ecca667330636ba0ca8ca5b6255fae7ea05627eee23326e461cfef27786", + "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011/build-pov-ray__gSzbLYX/agent/trajectory.json": "2a0e42851a8c3b0c8ee45f0dc3db1882eae3ba72cfd1fa883b181200d20588b3", + "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011/build-pov-ray__gSzbLYX/agent/bootstrap.json": "40256d355dfe43492278a1a4a22b8a6093657509086254ddd27494162c85b0b4", + "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011/build-pov-ray__gSzbLYX/verifier/retry-summary.json": "a617c5e8f570fde26f320b4ec71fe384e4a7eeb4e2831e160ddf13bdc51e90b2", + "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011/build-pov-ray__gSzbLYX/verifier/test-stdout.txt": "a073573c5c4f7813cea76ae6e4938a09def1ba1d871c86b6eba3261324fd3a37", + "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011/build-pov-ray__gSzbLYX/verifier/ctrf.json": "5fd556cc5936b1d417d20170cb2b5827ee41f14aa0bcffd558c8860099bb6a26", + "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011/build-pov-ray__gSzbLYX/verifier/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011-input/manifest.json": "7e912ca1a4aafc9d9f3cc31da8a0e72cdff0bc037ad8873b6945fd66881a0a6d", + "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011-input/config.json": "2b2b3247f92d5a56bebef93e7d8007be8ac26807533cdb8eed3f555af7c3bb52", + "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011-input/working-tree.patch": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "jobs/dev12-baseline-20261011-v3-orchestration/dev12-build-pov-ray-baseline-20261011.json": "a53545ba276d9b6fac51aec352d9a52d24e3169db5ef138361e13ef0f52bd4dd", + "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011/build-pov-ray__gSzbLYX/verifier/attempts/1/ctrf.json": "5fd556cc5936b1d417d20170cb2b5827ee41f14aa0bcffd558c8860099bb6a26", + "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011/build-pov-ray__gSzbLYX/verifier/attempts/1/reward.txt": "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865", + "jobs/dev12-baseline-main-source/jobs/dev12-build-pov-ray-baseline-20261011/build-pov-ray__gSzbLYX/verifier/attempts/1/test-stdout.txt": "a073573c5c4f7813cea76ae6e4938a09def1ba1d871c86b6eba3261324fd3a37" + } + } + ], + "post_run_pytorch_diagnostic": { + "kind": "post_run_diagnostic_not_benchmark_retry", + "fixture_url": "https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz", + "fixture_sha256": "8d422c7b0a1c1c79245a5bcf07fe86e33eeafee792b84584aec276f5a2dbc4e6", + "model_sha256": "a32103eef09936e586b1a7dc7e64d279500a38ff30fdd6df2167e28eb954eee6", + "method": "stdlib float64 forward pass; original packed float32 weights; original model.py ReLU architecture; compare first 50 images to official fixed expectations", + "expected_predictions": [ + 2, + 2, + 1, + 0, + 4, + 1, + 4, + 9, + 6, + 9, + 0, + 8, + 9, + 0, + 1, + 8, + 9, + 7, + 8, + 4, + 9, + 8, + 7, + 5, + 4, + 6, + 9, + 4, + 3, + 1, + 3, + 1, + 8, + 6, + 2, + 2, + 7, + 1, + 2, + 1, + 1, + 7, + 4, + 2, + 8, + 8, + 1, + 2, + 4, + 4 + ], + "variants": [ + { + "name": "raw01", + "mean": 0, + "std": 1, + "mismatch_count": 0, + "mismatch_indices": [], + "predictions": [ + 2, + 2, + 1, + 0, + 4, + 1, + 4, + 9, + 6, + 9, + 0, + 8, + 9, + 0, + 1, + 8, + 9, + 7, + 8, + 4, + 9, + 8, + 7, + 5, + 4, + 6, + 9, + 4, + 3, + 1, + 3, + 1, + 8, + 6, + 2, + 2, + 7, + 1, + 2, + 1, + 1, + 7, + 4, + 2, + 8, + 8, + 1, + 2, + 4, + 4 + ] + }, + { + "name": "mnist_standard", + "mean": 0.1307, + "std": 0.3081, + "mismatch_count": 10, + "mismatch_indices": [ + 0, + 18, + 21, + 22, + 25, + 26, + 28, + 32, + 34, + 44 + ], + "predictions": [ + 7, + 2, + 1, + 0, + 4, + 1, + 4, + 9, + 6, + 9, + 0, + 8, + 9, + 0, + 1, + 8, + 9, + 7, + 1, + 4, + 9, + 6, + 4, + 5, + 4, + 0, + 7, + 4, + 0, + 1, + 3, + 1, + 3, + 6, + 7, + 2, + 7, + 1, + 2, + 1, + 1, + 7, + 4, + 2, + 3, + 8, + 1, + 2, + 4, + 4 + ] + }, + { + "name": "agent_half", + "mean": 0.5, + "std": 0.5, + "mismatch_count": 14, + "mismatch_indices": [ + 0, + 11, + 15, + 18, + 21, + 22, + 25, + 26, + 28, + 32, + 33, + 34, + 44, + 45 + ], + "predictions": [ + 7, + 2, + 1, + 0, + 4, + 1, + 4, + 9, + 6, + 9, + 0, + 6, + 9, + 0, + 1, + 5, + 9, + 7, + 3, + 4, + 9, + 6, + 6, + 5, + 4, + 0, + 7, + 4, + 0, + 1, + 3, + 1, + 3, + 0, + 7, + 2, + 7, + 1, + 2, + 1, + 1, + 7, + 4, + 2, + 3, + 5, + 1, + 2, + 4, + 4 + ] + } + ] + }, + "final_image_integrity": { + "verified_at": "2026-10-11T17:22:55.161171+00:00", + "original_images_unchanged": 12, + "images": [ + { + "task": "terminal-bench/build-pov-ray", + "image": "alexgshaw/build-pov-ray:20251031", + "image_id": "sha256:874baf49341e2853a15a9184e308c7eb632e97f434359fcff2891c6e5ca1b39b", + "repo_digests": [ + "alexgshaw/build-pov-ray@sha256:874baf49341e2853a15a9184e308c7eb632e97f434359fcff2891c6e5ca1b39b" + ], + "size_bytes": 163066497 + }, + { + "task": "terminal-bench/cobol-modernization", + "image": "alexgshaw/cobol-modernization:20251031", + "image_id": "sha256:593ab9df3d83f771e927888b2d9436b380cdf7ec4d0af2a9f0802d541edebba0", + "repo_digests": [ + "alexgshaw/cobol-modernization@sha256:593ab9df3d83f771e927888b2d9436b380cdf7ec4d0af2a9f0802d541edebba0" + ], + "size_bytes": 162020401 + }, + { + "task": "terminal-bench/custom-memory-heap-crash", + "image": "alexgshaw/custom-memory-heap-crash:20251031", + "image_id": "sha256:1895d7d6240d5fb2ec774d6bfdd1057d170f8d199c15c582e4585d3c63802557", + "repo_digests": [ + "alexgshaw/custom-memory-heap-crash@sha256:1895d7d6240d5fb2ec774d6bfdd1057d170f8d199c15c582e4585d3c63802557" + ], + "size_bytes": 1267999268 + }, + { + "task": "terminal-bench/fix-code-vulnerability", + "image": "alexgshaw/fix-code-vulnerability:20251031", + "image_id": "sha256:cac325252991f823713b2d0441502972901dd782bd67f66c03d9b1e410dac5c0", + "repo_digests": [ + "alexgshaw/fix-code-vulnerability@sha256:cac325252991f823713b2d0441502972901dd782bd67f66c03d9b1e410dac5c0" + ], + "size_bytes": 190696453 + }, + { + "task": "terminal-bench/nginx-request-logging", + "image": "alexgshaw/nginx-request-logging:20251031", + "image_id": "sha256:693e64431a0dbf45952e8b91a298d54fd60247e501c6623fb8fa3ebc07fc3d3d", + "repo_digests": [ + "alexgshaw/nginx-request-logging@sha256:693e64431a0dbf45952e8b91a298d54fd60247e501c6623fb8fa3ebc07fc3d3d" + ], + "size_bytes": 67285778 + }, + { + "task": "terminal-bench/git-multibranch", + "image": "alexgshaw/git-multibranch:20251031", + "image_id": "sha256:5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5", + "repo_digests": [ + "alexgshaw/git-multibranch@sha256:5bc449921c138295f6ddd8dca2977319fee2ce523811dbe56cec4ca663385fb5" + ], + "size_bytes": 222134454 + }, + { + "task": "terminal-bench/db-wal-recovery", + "image": "alexgshaw/db-wal-recovery:20251031", + "image_id": "sha256:0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e", + "repo_digests": [ + "alexgshaw/db-wal-recovery@sha256:0e33ea5ec823975d1bd6c3778395c9f94251dd88f571146057bff6adb7e4594e" + ], + "size_bytes": 175276145 + }, + { + "task": "terminal-bench/multi-source-data-merger", + "image": "alexgshaw/multi-source-data-merger:20251031", + "image_id": "sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e", + "repo_digests": [ + "alexgshaw/multi-source-data-merger@sha256:8b32782078ff7383a1b4e5d3cecca8ce287e30f50bb4c0e5db009a18064e666e" + ], + "size_bytes": 220880128 + }, + { + "task": "terminal-bench/query-optimize", + "image": "alexgshaw/query-optimize:20251031", + "image_id": "sha256:28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14", + "repo_digests": [ + "alexgshaw/query-optimize@sha256:28de7da87004eab5b29bd5eaf1823579bc5b73ccea4b80349a5623185770da14" + ], + "size_bytes": 120494963 + }, + { + "task": "terminal-bench/adaptive-rejection-sampler", + "image": "alexgshaw/adaptive-rejection-sampler:20251031", + "image_id": "sha256:985bed39e8448bc95f138b5f766b59003a95c4be171c96af8e76448338f3dd8f", + "repo_digests": [ + "alexgshaw/adaptive-rejection-sampler@sha256:985bed39e8448bc95f138b5f766b59003a95c4be171c96af8e76448338f3dd8f" + ], + "size_bytes": 29731756 + }, + { + "task": "terminal-bench/constraints-scheduling", + "image": "alexgshaw/constraints-scheduling:20251031", + "image_id": "sha256:567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0", + "repo_digests": [ + "alexgshaw/constraints-scheduling@sha256:567ce5a189f8d11ac461790876e934cc7af38391baf89a78f95f4dafc1fec3b0" + ], + "size_bytes": 49852952 + }, + { + "task": "terminal-bench/pytorch-model-cli", + "image": "alexgshaw/pytorch-model-cli:20251031", + "image_id": "sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67", + "repo_digests": [ + "alexgshaw/pytorch-model-cli@sha256:c996884b5fb5fb2c0c07d67da2c4f974f271f07234ed28e1a868240375c57a67" + ], + "size_bytes": 44349611 + } + ] + }, + "publication_policy": "Only derived results and hashes are published. Raw logs remain local; the database trace contains a provider credential and must not be shared unredacted." +}