From e1be6ed1f81a17d73677b414d4d913adb81e9e25 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Mon, 5 Oct 2026 05:54:16 -0700 Subject: [PATCH 1/4] feat(agent): recover once from headless response truncation --- .../experiment.json | 150 +++++++++++++++ .../install-only.json | 27 +++ .../model-extraction.json | 22 +++ .../truncation-recovery-20261005/scheme.json | 22 +++ .../harbor/tests/test_atif_compatibility.py | 26 ++- docs/changelogs/0.10.x.md | 13 ++ docs/changelogs/README.md | 2 + docs/dev_docs/en/event-journal-protocol-v2.md | 6 +- docs/dev_docs/en/event-journal-protocol-v4.md | 23 ++- .../zh-CN/event-journal-protocol-v2.md | 3 +- .../zh-CN/event-journal-protocol-v4.md | 20 +- docs/user_docs/en/cli_reference.md | 18 +- docs/user_docs/zh-CN/cli_reference.md | 17 +- src/nanopycodeagent/agent.py | 34 +++- tests/test_truncation.py | 174 ++++++++++++++++-- 15 files changed, 516 insertions(+), 41 deletions(-) create mode 100644 benchmarks/harbor/configs/truncation-recovery-20261005/experiment.json create mode 100644 benchmarks/harbor/configs/truncation-recovery-20261005/install-only.json create mode 100644 benchmarks/harbor/configs/truncation-recovery-20261005/model-extraction.json create mode 100644 benchmarks/harbor/configs/truncation-recovery-20261005/scheme.json create mode 100644 docs/changelogs/0.10.x.md diff --git a/benchmarks/harbor/configs/truncation-recovery-20261005/experiment.json b/benchmarks/harbor/configs/truncation-recovery-20261005/experiment.json new file mode 100644 index 0000000..9fd827f --- /dev/null +++ b/benchmarks/harbor/configs/truncation-recovery-20261005/experiment.json @@ -0,0 +1,150 @@ +{ + "created_at": "2026-10-05T12:53:39.916356+00:00", + "question": "Can one bounded headless continuation after max_tokens preserve useful work and improve the two failed tasks without changing their budgets?", + "baseline_results": "benchmarks/harbor/results/tb21-v090-baseline-20261005.json", + "baseline_agent_ref": "c0605369d277352ac0d5ec20d02017b7cb149314", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "change": "One automatic truncation recovery per headless run. Keep the token cap and original deadline and turn count; skip truncated tools and incomplete thinking.", + "max_turns": 100, + "max_tokens": 65536, + "agent_setup_timeout_seconds": 1800, + "attempts_per_task": 1, + "whole_trial_retries": 0, + "execution": "Preflight both tasks, then launch the two disjoint single-task standard jobs together, at most two simultaneous trials. Each job retains n_concurrent_trials=2 but contains one task.", + "verifier_policy": "Original per-attempt limit; only verifier timeouts may retry, at most three attempts on the same agent output.", + "preflight": "Both task profiles are unsupported and must be recorded as not_configured; installation success does not certify verifier dependencies.", + "analysis": "Preserve every result. Count actual max_tokens responses and truncation_recovery injections. If no truncation occurs, a pass does not demonstrate recovery effectiveness. Report setup/verifier exceptions separately from scored failures. Do not replace the released pilot20 baseline or extrapolate a full-pilot score.", + "tasks": [ + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/model-extraction-relu-logits:20251031", + "docker_image_id": "sha256:52fe1f089f38650f0dc22d7531ee6e01ebd526de1b349aaa2ecb331dca9fabce", + "docker_repo_digests": [ + "alexgshaw/model-extraction-relu-logits@sha256:52fe1f089f38650f0dc22d7531ee6e01ebd526de1b349aaa2ecb331dca9fabce" + ], + "task_file_sha256": { + "instruction.md": "c9e179dd099065a34c362a865b0eb9fab5f7d96be582cf117e7519580c036ebe", + "task.toml": "eb64dfaf8ae88868911ae28855cdc4f767de464001e2704919015b595be6dc29", + "tests/test.sh": "3b2d8f5e0353c2bab3515d6387468a863a4f9f241ee131c73f545f3fdf9e2d52", + "tests/test_outputs.py": "e5189c7eb9942ede55e1b48a80388590b80a486b75497613f257b3373ad9b2c9" + } + }, + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6", + "native_agent_timeout_seconds": 2400, + "work_budget_seconds": 2220, + "verifier_timeout_seconds": 2400.0, + "docker_image": "alexgshaw/schemelike-metacircular-eval:20251031", + "docker_image_id": "sha256:b485be38bf80d56c1a4cd9da95bbf7ec129dc03f6786b86c0525518a96f6d1db", + "docker_repo_digests": [ + "alexgshaw/schemelike-metacircular-eval@sha256:b485be38bf80d56c1a4cd9da95bbf7ec129dc03f6786b86c0525518a96f6d1db" + ], + "task_file_sha256": { + "instruction.md": "3af5f9567b62f93683db7518ec6fcb609f83796258aa02166b5a7091c5b69031", + "task.toml": "8dae8eb226183b0ca92331c1106152a7abc93b03b4e5a95291b86a4ddc9b44e0", + "tests/interp.py": "efb8b5cdcf78b73afcf39d59600a1f355ca89b7bac9854bb01f110cc77bea2c1", + "tests/shadow_test/01-factorial.scm": "540e81fac476f04dc510ba107721d606ac980760088b2136889e7d4bf884e5c5", + "tests/shadow_test/02-fibonacci.scm": "628957171e0c4a3ee373635923fcd29e75bdf4bf13a402ac26fe94a5fc1443ac", + "tests/shadow_test/03-list-operations.scm": "fe8f17f59fe4045582fce363d52df95ba7c4de7345542c8bd4993a9f2f396235", + "tests/shadow_test/04-higher-order.scm": "9c2eba8c91bb7b224de56d789207d2eaad98bbe1f8c360d6eca4f014f42f9230", + "tests/shadow_test/05-simple-io.scm": "1eba30db30824fcf6048ea8e49b84e85c866cefce30862fce49b300e4416843c", + "tests/shadow_test/06-interactive-io.scm": "4fbc0e2d6e42c5a9495e549ccb13c02fcaeddf8fdaf8d2cf5de833116f12610a", + "tests/shadow_test/08-progn-sequencing.scm": "3bf8236dc5356d29ab8dbfe1b0f4c519354fbb23c8f3eaf946779c28b80feea3", + "tests/shadow_test/09-mutual-recursion.scm": "3bfa1f77e602c7039f8c8c70315e6d4fff6eed737dab0f0398e5bdf690137c2c", + "tests/shadow_test/10-advanced-features.scm": "b69ef497088fcf0aec388789a17561b783b8cdd5915719915a51c5ba189c31ff", + "tests/shadow_test/accumulator_patterns.scm": "8b9192796a6f9c3155cc5e44d0a0ba6a46d20a0d94e265b69b23ec8af43e4a07", + "tests/shadow_test/binary_tree.scm": "5a3dafc8844969cebb5809f253f4e54de7df480f378da8197f870badef03ee47", + "tests/shadow_test/church_numerals.scm": "31d075785f2f605bac3bd9ac2d99fcf7eaaed6686eb9afeab1d40bf5f35bbf2c", + "tests/shadow_test/closures.scm": "c7d025a376987f5ee22f529847b66eaf53f0af7033acbb4055cedb734168d612", + "tests/shadow_test/continuation_passing.scm": "79e7445c42227846f1adfaf08c17e278aac7a34df77482900b068be4fd248080", + "tests/shadow_test/currying.scm": "690a8cccbb1d249c8f6fcabc0193cecc5d0e4c1af39a0bb67c7e95fcff8e393e", + "tests/shadow_test/filter_operations.scm": "485a0563b81d002e368c5e4b2b7cc98a5306163701053630f416e3b783a2a238", + "tests/shadow_test/fold_operations.scm": "461b33db6b3f5580122d4c6dff0fd6d11c7f044865ffd684ade67d9b296a5a24", + "tests/shadow_test/function_composition.scm": "65ceee00188b12c3549134449e2f938d010c2c97dba090f5b53244abb3579b1f", + "tests/shadow_test/lazy_evaluation.scm": "edd95748de14aadf916c65209b5bcb8759d3e33b7ec935289a5128a656d33bec", + "tests/shadow_test/list_operations.scm": "660963851ecb49639f66557bd88db1e73db323186742145f33cb2e14d3af1103", + "tests/shadow_test/map_operations.scm": "974ac42d892f4b62b9b21ef7844581624dad811b38d35944f1005cc47f249856", + "tests/shadow_test/memoization.scm": "25482c1912fab1235651b1b7acd727f2ea6a55dc5c1919f841b44cfc32107b05", + "tests/shadow_test/mutual_recursion.scm": "2fbc9562e863cb2b325f86419fb2a7588265aaf5576295550722ee4274663712", + "tests/shadow_test/nested_defines.scm": "23ad3b2f936949501466da784511ea2e6215e2b198ca371bef86f6af877ca923", + "tests/shadow_test/oeis_sequences.scm": "1b586df6af2429a54808f7587f60021ba2b2c9bcc2ef2f9cbe44339699853c62", + "tests/shadow_test/oeis_sequences2.scm": "300da0b0dc20b883e01142adea9eb6da1a583a1b969ac0be47aa32c1ab66340e", + "tests/shadow_test/oeis_sequences3.scm": "dea198e874bf518b93c8406a843b64c1586cf4ce780fec8c444f8343e9322c75", + "tests/shadow_test/recursive_structures.scm": "64f1a430bf04a627f5e27bf004805544b60406f8a123d186516774dfcf3846cd", + "tests/shadow_test/test_read.scm": "fda448c22685e2f9af37ff7b986ee5e906d485a2db16f62eec113149f866870f", + "tests/shadow_test/variadic_functions.scm": "ac940ac0273e87ee06280fee192194ab5cdc14c88b61da9cfba92105e66b8f2f", + "tests/shadow_test/y_combinator.scm": "38d25b02d54342ab5610ad03fcef68c5f1d4eef8902295645f6acceb853c9860", + "tests/test/01-factorial.scm": "2623000a742c63d50ae1851854826247caee129c7d1613be6c7131d21f26a6ad", + "tests/test/02-fibonacci.scm": "db769cef446d5da9e9b1abccdf9ca644e239938095a98c21551aee25255a57d4", + "tests/test/03-list-operations.scm": "1a14b4100dedef30010fed62ea9fd295add1b0facb2e8e6e27c8d8eb3d90111b", + "tests/test/04-higher-order.scm": "59cd17bbffad641a37729b17c5bf9178b8bae7289fe89cc7135eb0bfec2b4fd6", + "tests/test/05-simple-io.scm": "3deeb3275df1b771449ebaebfcb9a934f375415e138c5e9f3685516c69a1f7a0", + "tests/test/06-interactive-io.scm": "aa3cae79b79dca90ac9d61a1babf64d6a5bb6559798c7727c11be02d8ddfd216", + "tests/test/08-progn-sequencing.scm": "4c0a4d980cf4223d1d17715d58adbe27dc120eced257a0d9f42de952c094bc5b", + "tests/test/09-mutual-recursion.scm": "939e18f4bd25b5ab0e2a4a767bf7eb6b0d8fb3530d28bf236f06cf27f2d90ee8", + "tests/test/10-advanced-features.scm": "c4cc5fb412df363875768c6a1e645255f6bc8f9ddf988d1ec2f4a266cedb8ec7", + "tests/test/README.md": "1f2174c7f12721f9f1898cda190c2886fb3ef9d35fd085f315da5814657104ae", + "tests/test/accumulator_patterns.scm": "50d246e00577d1f3ab3f0462c2b9618ea63680c33fcca12369cd1707b010cb5f", + "tests/test/binary_tree.scm": "c95f59ccac1ea4a9c846afddc993aa40ac081c0f9e6731544530109b269b67ae", + "tests/test/calculator.scm": "a09213ec8a2ec52486b9346c420ddb7c537576ef71b1cc5c6ec61d13d9139f61", + "tests/test/church_numerals.scm": "f37d858f32a9c5ba8868d094018c92516acadf0ea4e55b7c36f14320cb0c69a6", + "tests/test/closures.scm": "3ecfc35289479288876e20bf7f55e5b8eda84acd35937e89e3d062308a99aeff", + "tests/test/continuation_passing.scm": "01049581e582f7d35ef40ef68e1827d012da619beea57c1c74675a5f245318d7", + "tests/test/currying.scm": "2844d54eee9a94af05a11b4aae06d6e7823d193f17235c8f90bf98926cbca75d", + "tests/test/filter_operations.scm": "f6bc75f6da9ece96c0616e3570e4df59d0f2eb4655375d3a46b0349c86cc5dd8", + "tests/test/fold_operations.scm": "05aab492ad914baecd19cf454ed587e7a031e94b8990500eb1c869c15da6fc0b", + "tests/test/function_composition.scm": "9b56dc78823e7d35adb213744c0f8e95684074d6a12bb98c8c05755788be5b06", + "tests/test/lazy_evaluation.scm": "ea5912a53a35b65bf2bee9483dca90814b8baf52ae3ae50c2d4589f794dca54d", + "tests/test/list_operations.scm": "d7e70621afbbb787cf646e2089bccfbd3681fbb2e70325f3e50f56ccd30dbd9d", + "tests/test/map_operations.scm": "0cebddb3d4a50e4b6f6bfc0e8c12762d3f42609030aef077d8ab4bedda4718d9", + "tests/test/memoization.scm": "cf8202df22407dbba72ca7f660d8ab7cd4ca962b215c36cc491d5c1a474c160b", + "tests/test/mutual_recursion.scm": "a3dc23a96ec1cf05b0be4a4f0b6d3da9e2bad1aa3f7a83c7566f71f980a67fe2", + "tests/test/nested_defines.scm": "21faa0ccaf744cb8c9d283bbfd0c36e195c803c85f511e70a99267c0a85b6216", + "tests/test/oeis_sequences.scm": "d1f77161b367176b3dfab55a34466e511c602c7ae359e05f5a715627b93dca6a", + "tests/test/oeis_sequences2.scm": "5679ef0f99743114d8c06edcdcde1fea9474b71aeb57c2069f1ee33df1970bce", + "tests/test/oeis_sequences3.scm": "5dc8b44158b2648a6e9766f6bcdf21553a07bf487d79e64ebb418511ef3b8186", + "tests/test/recursive_structures.scm": "67838ff5bac26ea359139720adec1d5700176147b0062065c7def9715e8a2aeb", + "tests/test/test_read.scm": "5b9ee381c3cf45b2fc8c2e7b490857fb67f1ac78653b586d7f2848daf6853809", + "tests/test/variadic_functions.scm": "c9b847c50840c6977904e5033a07f6108cc3a54012b02d916e79025fb43e0f6e", + "tests/test/y_combinator.scm": "971144dcb2b04aaa3b34d9279459bd7d2e8256c585d17809465848c2589d36d8", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "fbb84e3c0c2f72ed7de90f54380f860052c16155e7d0c9076cd57c966e9be7ed" + } + } + ], + "source_sha256": { + "src/nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "src/nanopycodeagent/agent.py": "8e28a5c3fcd140441484c46d1e62ec811d325aaff484141a4b9837baead6fc52", + "src/nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "src/nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "src/nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "src/nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "src/nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "src/nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "src/nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "src/nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "src/nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "src/nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "src/nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "src/nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "src/nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "benchmarks/harbor/src/harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "benchmarks/harbor/src/harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "benchmarks/harbor/src/harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "benchmarks/harbor/src/harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "benchmarks/harbor/src/harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "benchmarks/harbor/src/harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "configuration_sha256": { + "model-extraction.json": "f8baf52e182e6b6ab669644c904c1c961db5e6083c6c18b8d1479932a9b26588", + "scheme.json": "0bf57205bb0d82ea0342483a269eb995441a7c427fec40f0678f3a69b1a87b0b", + "install-only.json": "75e6b3f9d567c1c2db953deef82cc43ec8ea31c3c79ab15fa11599c68531aaaf" + } +} diff --git a/benchmarks/harbor/configs/truncation-recovery-20261005/install-only.json b/benchmarks/harbor/configs/truncation-recovery-20261005/install-only.json new file mode 100644 index 0000000..2c71e92 --- /dev/null +++ b/benchmarks/harbor/configs/truncation-recovery-20261005/install-only.json @@ -0,0 +1,27 @@ +{ + "n_concurrent_trials": 2, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17" + }, + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6" + } + ], + "install_only": true +} diff --git a/benchmarks/harbor/configs/truncation-recovery-20261005/model-extraction.json b/benchmarks/harbor/configs/truncation-recovery-20261005/model-extraction.json new file mode 100644 index 0000000..044f999 --- /dev/null +++ b/benchmarks/harbor/configs/truncation-recovery-20261005/model-extraction.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 2, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17" + } + ] +} diff --git a/benchmarks/harbor/configs/truncation-recovery-20261005/scheme.json b/benchmarks/harbor/configs/truncation-recovery-20261005/scheme.json new file mode 100644 index 0000000..0c80195 --- /dev/null +++ b/benchmarks/harbor/configs/truncation-recovery-20261005/scheme.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 2, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 2220 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6" + } + ] +} diff --git a/benchmarks/harbor/tests/test_atif_compatibility.py b/benchmarks/harbor/tests/test_atif_compatibility.py index d4ca37b..c98bbd5 100644 --- a/benchmarks/harbor/tests/test_atif_compatibility.py +++ b/benchmarks/harbor/tests/test_atif_compatibility.py @@ -134,7 +134,8 @@ def test_recovered_stream_v3_passes_harbor_atif_validator(tmp_path, resolved): [{"type": "tool_call", "tool_call_id": "call-1", "tool_name": "write", "input": {}}], [], ]) -def test_truncated_v2_journal_passes_harbor_atif_validator(tmp_path, content): +@pytest.mark.parametrize("recover", [False, True]) +def test_truncated_journal_passes_harbor_atif_validator(tmp_path, content, recover): fixture = Path(__file__).parent / "fixtures" / "atif-journal-v1.jsonl" with EventJournal.create("run-truncated", directory=tmp_path) as journal: for entry in EventJournal.replay(fixture): @@ -146,12 +147,31 @@ def test_truncated_v2_journal_passes_harbor_atif_validator(tmp_path, content): "stop_reason": "max_tokens", "content": content, "tool_calls": [block for block in content if block["type"] == "tool_call"], } + truncated_payload = payload elif entry.type == "run.completed": - payload = payload | {"outcome": "response_truncated"} + if recover: + journal.append(NativeEvent("input.injected", { + "model_call_id": "recovered", "content": "Continue within the remaining budget.", + "reason": "truncation_recovery", "source_timestamp": payload["source_timestamp"], + })) + journal.append(NativeEvent("model.started", { + "model_call_id": "recovered", "model": truncated_payload["model"], + "source_timestamp": payload["source_timestamp"], + })) + journal.append(NativeEvent("model.completed", truncated_payload | { + "model_call_id": "recovered", "message_id": "msg-recovered", + "content": [{"type": "text", "text": "done"}], "tool_calls": [], + "stop_reason": "end_turn", + })) + payload = payload | {"outcome": "completed" if recover else "response_truncated"} journal.append(NativeEvent(entry.type, payload)) trajectory = project_atif(EventJournal.replay(journal.path)) validator = TrajectoryValidator() assert validator.validate(trajectory), validator.get_errors() - assert trajectory["extra"]["terminal"]["outcome"] == "response_truncated" + assert trajectory["extra"]["terminal"]["outcome"] == ("completed" if recover else "response_truncated") assert "observation" not in trajectory["steps"][1] + assert trajectory["steps"][1]["extra"]["stop_reason"] == "max_tokens" + if recover: + assert trajectory["steps"][2]["extra"]["reason"] == "truncation_recovery" + assert trajectory["steps"][3]["extra"]["stop_reason"] == "end_turn" diff --git a/docs/changelogs/0.10.x.md b/docs/changelogs/0.10.x.md new file mode 100644 index 0000000..6a65c84 --- /dev/null +++ b/docs/changelogs/0.10.x.md @@ -0,0 +1,13 @@ +# Changelog — 0.10.x + +All notable changes in the **0.10.x** release series are documented here. + +## [Unreleased] + +### Changed +- Allow headless tasks to recover once after a response reaches `max_tokens`, + within the original time and reply budgets and without raising the token + limit. Skip all tools from the truncated response and remove unfinished + thinking from follow-up requests. Preserve per-attempt usage and costs, and + record the recovery prompt in Journal and ATIF. A second truncation still + stops the run; interactive mode continues to wait for user input. diff --git a/docs/changelogs/README.md b/docs/changelogs/README.md index 81dd38d..851ebbe 100644 --- a/docs/changelogs/README.md +++ b/docs/changelogs/README.md @@ -8,3 +8,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/). Current release series: [0.9.x](0.9.x.md). Previous release series: [0.8.x](0.8.x.md). + +Upcoming development: [0.10.x](0.10.x.md). diff --git a/docs/dev_docs/en/event-journal-protocol-v2.md b/docs/dev_docs/en/event-journal-protocol-v2.md index 18fb2ea..4d35480 100644 --- a/docs/dev_docs/en/event-journal-protocol-v2.md +++ b/docs/dev_docs/en/event-journal-protocol-v2.md @@ -32,8 +32,10 @@ tools. A reply that also spends the last turn therefore records `response_truncated`. All tools in that reply are skipped: there are no `tool.started` or `tool.completed` events and no fabricated observations. Existing text remains on stdout, the truncation diagnostic goes to stderr, -and headless mode exits `0`. The current policy stops without automatically -continuing, retrying, or raising the 8192-token limit. +and headless mode exits `0`. The policy introduced with v2 stopped without automatically +continuing, retrying, or raising the then-current 8192-token limit. Later +[headless recovery](event-journal-protocol-v4.md#bounded-truncation-recovery) can +continue once within the original budgets. This is a budget outcome with an explicit reason, represented by `run.completed` rather than `run.failed`. ATIF's `extra.terminal.status` diff --git a/docs/dev_docs/en/event-journal-protocol-v4.md b/docs/dev_docs/en/event-journal-protocol-v4.md index f4c37f4..a2720fa 100644 --- a/docs/dev_docs/en/event-journal-protocol-v4.md +++ b/docs/dev_docs/en/event-journal-protocol-v4.md @@ -54,9 +54,9 @@ payload fields are: | Field | Type | Meaning | | --- | --- | --- | -| `model_call_id` | nonempty string | The upcoming model attempt that will use this input, matching the immediately following `model.started`. | +| `model_call_id` | nonempty string | The upcoming model attempt that will use this input, matching the next `model.started`. | | `content` | string | The complete text appended this time, not a copy of the whole conversation or tool results. | -| `reason` | nonempty string | Reason for injection: `time_budget` when a time budget is configured, or `turn_budget` when only the reply count is limited. | +| `reason` | nonempty string | Reason for injection: `time_budget` when a time budget is configured, `turn_budget` when only the reply count is limited, or `truncation_recovery` for the single automatic headless continuation. | | `source_timestamp` | RFC 3339 UTC or null | Time of injection. | With a time budget configured, every model attempt, including retries, appends @@ -83,6 +83,25 @@ observations, and does not add to `llm_call_count`. metadata is projected into `extra.journal_truncation`. Correlation identifiers and `reason` are exempt from string truncation. +## Bounded truncation recovery + +Headless runs can continue once after `stop_reason="max_tokens"`, provided a +reply and time remain within their original budgets. The truncated reply still +has its own `model.completed` record, usage, and cost. Its tools do not execute; +follow-up request history keeps only visible text and a truncation notice. + +The continuation adds a user-role message and records `input.injected` with +`reason="truncation_recovery"`, associated with the next model attempt. A budget +reminder can follow before that attempt's `model.started`. A transport retry +reuses the recovery message without injecting it again. Successful tool work or +transport retries do not renew the once-per-run allowance. A later truncation +ends with `response_truncated`; normal completion or budget exhaustion retains +its existing outcome. Interactive runs still return to the user after truncation. + +This adds no event type, outcome, or required field: `reason` is an open string, +so schema v4 and existing readers remain compatible. ATIF preserves the injected +message and both model steps, including the first step's `max_tokens` reason. + ## Compatibility and validation Outcomes and event types are closed enums; new values require a schema diff --git a/docs/dev_docs/zh-CN/event-journal-protocol-v2.md b/docs/dev_docs/zh-CN/event-journal-protocol-v2.md index f5e3c45..4b1c90e 100644 --- a/docs/dev_docs/zh-CN/event-journal-protocol-v2.md +++ b/docs/dev_docs/zh-CN/event-journal-protocol-v2.md @@ -25,7 +25,8 @@ usage 和已有 provider 标识。正常费用补查仍在 run 收尾时执行 core 在检查轮数上限和执行工具之前识别截断，即使该回复恰好用尽最后一轮，也记录 `response_truncated`。该回复中的工具一律不执行，不产生 `tool.started` 或 `tool.completed`，也不伪造 observation。已有文本继续保留在 stdout，截断提示写入 -stderr；headless 退出码为 `0`。当前策略是停止，不自动续写、重试或增大 8192 上限。 +stderr；headless 退出码为 `0`。v2 引入时的策略是停止，不自动续写、重试或增大当时的 8192 上限。后续的 +[headless 恢复策略](event-journal-protocol-v4.md#有界的截断恢复)允许在原始预算内继续一次。 这属于有明确原因的预算终态，使用 `run.completed`，而非 `run.failed`。 ATIF 的 `extra.terminal.status` 仍为 `completed`，表示 run 已正常收尾； diff --git a/docs/dev_docs/zh-CN/event-journal-protocol-v4.md b/docs/dev_docs/zh-CN/event-journal-protocol-v4.md index 30a07f4..c2a8ccc 100644 --- a/docs/dev_docs/zh-CN/event-journal-protocol-v4.md +++ b/docs/dev_docs/zh-CN/event-journal-protocol-v4.md @@ -42,9 +42,9 @@ | 字段 | 类型 | 含义 | | --- | --- | --- | -| `model_call_id` | 非空字符串 | 即将使用该输入的模型尝试标识，与紧随其后的 `model.started` 对应。 | +| `model_call_id` | 非空字符串 | 即将使用该输入的模型尝试标识，与后续最近的 `model.started` 对应。 | | `content` | 字符串 | 本次追加的完整文本，不是整个对话或工具结果的副本。 | -| `reason` | 非空字符串 | 注入原因；配置时间预算时使用 `time_budget`，只有轮数预算时使用 `turn_budget`。 | +| `reason` | 非空字符串 | 注入原因；配置时间预算时使用 `time_budget`，只有轮数预算时使用 `turn_budget`；单次 headless 自动续行使用 `truncation_recovery`。 | | `source_timestamp` | RFC 3339 UTC 或 null | 注入发生时间。 | 配置时间预算时，每次模型尝试（包括重试）都会将提醒追加到最近的用户消息尾部， @@ -65,6 +65,22 @@ observation，也不增加 `llm_call_count`。 `content` 沿用 Journal 字符串持久化上限，发生截断时将相关元数据投影到 `extra.journal_truncation`。关联标识和 `reason` 不受字符串截断影响。 +## 有界的截断恢复 + +Headless run 收到 `stop_reason="max_tokens"` 后，如果原始回合与时间预算尚有 +剩余，可以自动继续一次。截断回复仍保留自己的 `model.completed`、usage 和费用。 +该回复的工具不会执行；后续请求历史只保留可见文本和截断提示。 + +恢复时增加一条用户角色消息，并记录 reason 为 `truncation_recovery` 的 +`input.injected`，关联下一次模型尝试。在该次 `model.started` 之前还可能追加 +预算提醒。传输重试复用恢复消息，不再次注入。成功的工具执行或传输重试都不会 +重置每次 run 仅一次的恢复额度。后续再次截断则以 `response_truncated` 结束； +正常完成和预算耗尽仍使用原有 outcome。交互模式在截断后仍返回用户输入。 + +没有新增事件类型、outcome 或必填字段：`reason` 是开放字符串，因此保持 schema +v4，与已有 reader 兼容。ATIF 保留注入消息与两次模型步骤，包括首次步骤的 +`max_tokens` 原因。 + ## 兼容性与验证 outcome 和事件类型都是封闭枚举，新增值需要递增 schema。v1/v2/v3 记录包含 diff --git a/docs/user_docs/en/cli_reference.md b/docs/user_docs/en/cli_reference.md index 4dd0710..cbf9747 100644 --- a/docs/user_docs/en/cli_reference.md +++ b/docs/user_docs/en/cli_reference.md @@ -105,11 +105,19 @@ default. The provider must accept the requested limit for the chosen model. The limit applies to the whole generated reply, including thinking where the provider counts it; it is not a guaranteed amount of visible answer text. -If the provider -returns `stop_reason="max_tokens"`, the agent stops that run, prints a truncation -diagnostic to stderr, and skips all tools from that reply. It preserves partial -text and records `response_truncated` as the trajectory terminal outcome. -Headless mode still exits `0`; it does not retry or continue automatically. +If the provider returns `stop_reason="max_tokens"`, the agent prints a truncation +diagnostic to stderr and skips all tools from that reply. Headless mode can +automatically continue **once per run** if time and reply budgets remain. The +follow-up retains partial visible text and a notice, discards unexecuted tool +calls and incomplete thinking, and asks the model to continue from confirmed +tool results and current files. It keeps the same token limit and original +deadline; the truncated reply and recovery both consume the reply budget. + +A second truncation, or truncation on the last allowed reply, ends with +`response_truncated`; an expired deadline ends with `time_budget_exhausted`. +Headless mode still exits `0` for these budget outcomes. Recovery is recorded as +an `input.injected` event with reason `truncation_recovery`, while every model +attempt retains its own stop reason, usage, and cost in the Journal and ATIF. Interactive mode returns to `You>` with the partial text and a truncation notice in conversation history, so a later user message can continue the conversation. diff --git a/docs/user_docs/zh-CN/cli_reference.md b/docs/user_docs/zh-CN/cli_reference.md index 6fc1907..fa35a1f 100644 --- a/docs/user_docs/zh-CN/cli_reference.md +++ b/docs/user_docs/zh-CN/cli_reference.md @@ -92,11 +92,18 @@ nanoPyCodeAgent -p "fix the failing tests" 它限制整次生成；provider 将 thinking 计入生成预算时，thinking 也占用这一上限， 因此它不保证相同数量的可见回答文本。 -如果 provider 返回 -`stop_reason="max_tokens"`,agent 会停止本次 run,向 stderr 打印截断诊断,并跳过该 -回复中的所有工具调用。已输出的文本会保留,trajectory 的终态 outcome 记录为 -`response_truncated`。Headless 模式仍退出 `0`,不会自动重试或续写。交互模式会 -返回 `You>`,会话历史中保留部分文本和截断提示,用户可以在后续消息中继续对话。 +如果 provider 返回 `stop_reason="max_tokens"`，agent 会向 stderr 打印截断诊断， +并跳过该回复中的所有工具调用。时间与回合预算尚有剩余时，headless 模式**每次 +run 最多自动恢复一次**。后续请求保留可见的部分文本和提示，移除未执行的工具 +调用及不完整的 thinking，并要求模型从已确认的工具结果和当前文件继续。token +上限与原始截止时间不变；截断回复与恢复回复都计入回合预算。 + +第二次截断，或最后一个允许回合发生截断时，终态为 `response_truncated`；截止 +时间已到时，终态为 `time_budget_exhausted`。Headless 模式对这些预算终态仍退出 +`0`。恢复提示记录为 reason 为 `truncation_recovery` 的 `input.injected` 事件， +每次模型尝试的 stop reason、usage 和费用均保留在 Journal 与 ATIF 中。 +交互模式会返回 `You>`，会话历史中保留部分文本和截断提示，用户可以在后续消息 +中继续对话。 ## 响应中断 diff --git a/src/nanopycodeagent/agent.py b/src/nanopycodeagent/agent.py index d7773a9..3be9bf9 100644 --- a/src/nanopycodeagent/agent.py +++ b/src/nanopycodeagent/agent.py @@ -75,9 +75,15 @@ STREAM_RETRY_WINDOW_SECONDS = 300.0 _TRUNCATION_NOTICE = ( - "[response truncated: reached max_tokens; stopped without finishing the task. " + "[response truncated: reached max_tokens before finishing the task. " "Tool calls from this response were not executed.]" ) +_TRUNCATION_RECOVERY_NOTE = ( + "The previous response reached its output limit without finishing the task. " + "No tool calls from that response were executed. Continue from the last " + "confirmed tool results and current files within the remaining budget. " + "This is the only automatic truncation recovery for this run." +) # How many model replies one headless task may spend before the run stops on # its own. The interactive loop needs no such cap — a human watching the @@ -298,6 +304,8 @@ def __call__(self, event: NativeEvent) -> None: print() if event.payload["stop_reason"] == "max_tokens": print(_TRUNCATION_NOTICE, file=sys.stderr) + elif event.type == "input.injected" and event.payload["reason"] == "truncation_recovery": + print("[continuing after response truncation; recovery 1/1]", file=sys.stderr) elif event.type == "model.failed" and event.payload["will_retry"]: if str(event.payload["model_call_id"]) in self._model_calls_with_text: print() @@ -490,7 +498,8 @@ def _run_exchange( Appends assistant replies and tool results to ``messages`` in place. A truncated reply retains only its text and a notice in request history; - the original response is kept in the journal. Returns the stopping outcome, + the original response is kept in the journal. Headless runs may recover + once within the original budgets. Returns the stopping outcome, distinguishing completion, turn-budget exhaustion, and response truncation. """ run_id = f"run-{uuid.uuid4()}" @@ -603,6 +612,8 @@ def _run_model_loop( turns = 0 retries = 0 retry_deadline = None + truncation_recovered = False + recovery_pending = False deadline = ( time.monotonic() + time_budget_seconds if time_budget_seconds is not None and time_budget_seconds > 0 @@ -615,6 +626,15 @@ def _run_model_loop( remaining = deadline - time.monotonic() if remaining <= 0: return "time_budget_exhausted" + if recovery_pending: + messages.append({"role": "user", "content": _TRUNCATION_RECOVERY_NOTE}) + emitter.emit("input.injected", { + "model_call_id": model_call_id, + "content": _TRUNCATION_RECOVERY_NOTE, + "reason": "truncation_recovery", + "source_timestamp": utc_now(), + }) + recovery_pending = False if deadline is not None or (max_turns is not None and retries == 0): _append_budget_note( messages, @@ -784,7 +804,7 @@ def _run_model_loop( if message.stop_reason == "max_tokens": # Do not execute partial tool calls or replay them without results. # Thinking may also be cut off before its signature arrives. Keep - # visible text and an explicit notice for the next interactive turn. + # visible text and an explicit notice for a safe follow-up request. text = "".join( block.text for block in message.content if block.type == "text" ) @@ -794,6 +814,14 @@ def _run_model_loop( "content": (f"{text}\n\n" if text else "") + _TRUNCATION_NOTICE, } ) + if ( + not truncation_recovered + and (max_turns is not None or time_budget_seconds is not None) + and (max_turns is None or turns < max_turns) + ): + truncation_recovered = True + recovery_pending = True + continue return "response_truncated" request_content = [] for block in message.content: diff --git a/tests/test_truncation.py b/tests/test_truncation.py index 32222a0..e4452a1 100644 --- a/tests/test_truncation.py +++ b/tests/test_truncation.py @@ -18,6 +18,7 @@ patch_client_and_input, sdk_http_module, text_block, + tool_use_block, write_tool_use_block, ) @@ -38,13 +39,16 @@ def _journal_entries(): def test_truncation_stops_with_usage_cost_and_distinct_terminal( monkeypatch, tmp_path, capsys, content, max_turns, max_tokens ): - reply = FakeStream( - content, - stop_reason="max_tokens", - usage=SimpleNamespace(input_tokens=10, output_tokens=max_tokens), - response_headers={"x-generation-id": "gen-truncated"}, - ) - messages = FakeMessages([reply]) + attempts = 1 if max_turns == 1 else 2 + messages = FakeMessages([ + FakeStream( + content, + stop_reason="max_tokens", + usage=SimpleNamespace(input_tokens=10, output_tokens=max_tokens), + response_headers={"x-generation-id": f"gen-truncated-{index}"}, + ) + for index in range(attempts) + ]) patch_client(monkeypatch, FakeClient(messages)) reconciled = [] @@ -65,14 +69,15 @@ def resolve(base_url, generation_id, credential, **kwargs): "--trajectory", str(trajectory_path), ]) == 0 - assert len(messages.calls) == 1 - assert messages.kwargs[0]["max_tokens"] == max_tokens + assert len(messages.calls) == attempts + assert all(call["max_tokens"] == max_tokens for call in messages.kwargs) captured = capsys.readouterr() - assert captured.out == ("Partial answer\n" if content and content[0].type == "text" else "") + assert captured.out == ("Partial answer\n" * attempts if content and content[0].type == "text" else "") assert "response truncated" in captured.err assert "max_tokens" in captured.err assert "stopped after" not in captured.err - assert reconciled == ["gen-truncated"] + assert captured.err.count("recovery 1/1") == attempts - 1 + assert reconciled == [f"gen-truncated-{index}" for index in range(attempts)] entries = _journal_entries() assert all(entry.schema_version == 4 for entry in entries) @@ -88,10 +93,122 @@ def resolve(base_url, generation_id, credential, **kwargs): assert trajectory["schema_version"] == "ATIF-v1.7" assert trajectory["extra"]["terminal"]["outcome"] == "response_truncated" assert next(step for step in trajectory["steps"] if step["source"] == "agent")["extra"]["stop_reason"] == "max_tokens" - assert trajectory["final_metrics"]["total_prompt_tokens"] == 10 + assert trajectory["final_metrics"]["total_prompt_tokens"] == 10 * attempts assert trajectory["agent"]["extra"]["max_tokens"] == max_tokens - assert trajectory["final_metrics"]["total_completion_tokens"] == max_tokens - assert trajectory["final_metrics"]["total_cost_usd"] == 0.01 + assert trajectory["final_metrics"]["total_completion_tokens"] == max_tokens * attempts + assert trajectory["final_metrics"]["total_cost_usd"] == 0.01 * attempts + + +def test_headless_recovery_keeps_committed_work_and_discards_truncated_tools( + monkeypatch, tmp_path +): + unsafe = tmp_path / "must-not-exist.txt" + answer = tmp_path / "answer.txt" + executed = [] + monkeypatch.setattr(agent, "run_bash", lambda command, **kwargs: (executed.append(command) or "saved", False)) + + def reply(content, reason="tool_use", tokens=10): + return FakeStream(content, stop_reason=reason, + usage=SimpleNamespace(input_tokens=10, output_tokens=tokens, cost=0.01)) + + messages = FakeMessages([ + reply([tool_use_block("committed", "append once")]), + reply([ + ThinkingBlock(type="thinking", thinking="Unfinished thinking", signature=""), + text_block("Partial reply"), + write_tool_use_block("discard-complete", path=str(unsafe), content="unsafe"), + write_tool_use_block("discard-partial", path=str(unsafe)), + ], "max_tokens", 8192), + reply([write_tool_use_block("fresh", path=str(answer), content="done")]), + reply([text_block("Finished")], "end_turn"), + ]) + patch_client(monkeypatch, FakeClient(messages)) + trajectory_path = tmp_path / "trajectory.json" + assert agent.run_headless("task", max_turns=4, max_tokens=8192, trajectory_path=trajectory_path) == 0 + assert executed == ["append once"] + assert not unsafe.exists() + assert answer.read_text() == "done" + followup = messages.calls[2] + assert [m["role"] for m in followup] == ["user", "assistant", "user", "assistant", "user"] + assert followup[-2]["content"].startswith("Partial reply\n\n") + assert "not executed" in followup[-2]["content"] + assert agent._TRUNCATION_RECOVERY_NOTE in followup[-1]["content"] + assert "Turn 3 of 4" in followup[-1]["content"] + assert "Unfinished thinking" not in str(followup) + assert "discard-complete" not in str(followup) + assert "discard-partial" not in str(followup) + assert all(call["max_tokens"] == 8192 for call in messages.kwargs) + assert len({call["system"] for call in messages.kwargs}) == 1 + + entries = _journal_entries() + injected = [e for e in entries if e.type == "input.injected" and e.payload["reason"] == "truncation_recovery"] + assert len(injected) == 1 + starts = [e for e in entries if e.type == "model.started"] + assert injected[0].payload["model_call_id"] == starts[2].payload["model_call_id"] + assert [e.payload["tool_call_id"] for e in entries if e.type == "tool.started"] == ["committed", "fresh"] + trajectory = json.loads(trajectory_path.read_text()) + assert trajectory["extra"]["terminal"]["outcome"] == "completed" + recoveries = [s for s in trajectory["steps"] if s.get("extra", {}).get("reason") == "truncation_recovery"] + assert len(recoveries) == 1 and recoveries[0]["message"] == agent._TRUNCATION_RECOVERY_NOTE + assert trajectory["final_metrics"]["total_prompt_tokens"] == 40 + assert trajectory["final_metrics"]["total_completion_tokens"] == 8222 + assert trajectory["final_metrics"]["total_cost_usd"] == 0.04 + + +def test_recovery_allowance_is_not_reset_by_tool_work_or_stream_retry(monkeypatch): + import httpx + from test_stream_recovery import BrokenStream + + monkeypatch.setattr(agent.time, "sleep", lambda _: None) + executed = [] + monkeypatch.setattr(agent, "run_bash", lambda command, **kwargs: (executed.append(command) or "ok", False)) + messages = FakeMessages([ + FakeStream([], stop_reason="max_tokens"), + BrokenStream(httpx.ReadError("interrupted recovery")), + FakeStream([tool_use_block("fresh", "work")], stop_reason="tool_use"), + FakeStream([], stop_reason="max_tokens"), + ]) + patch_client(monkeypatch, FakeClient(messages)) + assert agent.run_headless("task", max_turns=10) == 0 + assert len(messages.calls) == 4 + assert messages.calls[1] == messages.calls[2] + assert executed == ["work"] + entries = _journal_entries() + assert entries[-1].payload["outcome"] == "response_truncated" + assert sum(e.type == "input.injected" and e.payload["reason"] == "truncation_recovery" for e in entries) == 1 + + +def test_recovery_cannot_bypass_last_turn_tool_guard(monkeypatch, tmp_path): + target = tmp_path / "must-not-exist.txt" + messages = FakeMessages([ + FakeStream([], stop_reason="max_tokens"), + FakeStream([write_tool_use_block("last", path=str(target), content="late")], stop_reason="tool_use"), + ]) + patch_client(monkeypatch, FakeClient(messages)) + assert agent.run_headless("task", max_turns=2) == 0 + assert len(messages.calls) == 2 + assert not target.exists() + assert _journal_entries()[-1].payload["outcome"] == "max_turns_exhausted" + + +@pytest.mark.parametrize("budget, expected_calls", [(4, 1), (6, 2)]) +def test_recovery_shares_the_original_deadline(monkeypatch, budget, expected_calls): + from test_time_budget import AdvancingFakeMessages, FakeTime + + clock = FakeTime() + monkeypatch.setattr(agent, "time", clock) + messages = AdvancingFakeMessages([ + FakeStream([], stop_reason="max_tokens"), + FakeStream([text_block("late")]), + ], clock, 4) + patch_client(monkeypatch, FakeClient(messages)) + assert agent.run_headless("task", time_budget_seconds=budget) == 0 + assert len(messages.calls) == expected_calls + assert messages.kwargs[0]["timeout"] == budget + if expected_calls == 2: + assert messages.kwargs[1]["timeout"] == 2 + assert "Turn 2 of 50" in messages.calls[1][-1]["content"] + assert _journal_entries()[-1].payload["outcome"] == "time_budget_exhausted" def test_truncated_tools_are_not_executed_or_replayed_on_the_next_user_turn( @@ -128,8 +245,9 @@ def test_truncated_tools_are_not_executed_or_replayed_on_the_next_user_turn( assert captured.err.count("response truncated") == 1 +@pytest.mark.parametrize("recover", [False, True]) def test_sdk_stream_with_partial_tool_json_preserves_truncation( - monkeypatch, tmp_path, capsys + monkeypatch, tmp_path, capsys, recover ): """Use the real SDK accumulator with an input JSON delta cut mid-string.""" httpx = sdk_http_module() @@ -156,8 +274,25 @@ def test_sdk_stream_with_partial_tool_json_preserves_truncation( def respond(request): requests.append(request) + if recover and len(requests) == 2: + completed = [ + {"type": "message_start", "message": { + "id": "msg-recovered", "type": "message", "role": "assistant", + "model": "test-model", "content": [], "stop_reason": None, + "stop_sequence": None, "usage": {"input_tokens": 20, "output_tokens": 0}, + }}, + {"type": "content_block_start", "index": 0, + "content_block": {"type": "text", "text": "Done"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 1}}, + {"type": "message_stop"}, + ] + body = "".join(f"event: {e['type']}\ndata: {json.dumps(e)}\n\n" for e in completed) + else: + body = wire return httpx.Response( - 200, headers={"content-type": "text/event-stream"}, content=wire + 200, headers={"content-type": "text/event-stream"}, content=body ) with anthropic.Anthropic( @@ -169,11 +304,14 @@ def respond(request): "write a file", max_tokens=65536, trajectory_path=tmp_path / "trajectory.json" ) == 0 - assert len(requests) == 1 + assert len(requests) == 2 assert json.loads(requests[0].content)["max_tokens"] == 65536 + followup = json.loads(requests[1].content)["messages"] + assert isinstance(followup[-2]["content"], str) + assert "partial-tool" not in json.dumps(followup) assert "response truncated" in capsys.readouterr().err entries = _journal_entries() - assert entries[-1].payload["outcome"] == "response_truncated" + assert entries[-1].payload["outcome"] == ("completed" if recover else "response_truncated") completed = next(entry for entry in entries if entry.type == "model.completed") assert completed.payload["tool_calls"][0]["input"] == {} assert not any(entry.type.startswith("tool.") for entry in entries) From e14056aa92a249f90c25ce48eba6efb7c552ceb4 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Mon, 5 Oct 2026 06:27:26 -0700 Subject: [PATCH 2/4] docs(benchmarks): record the truncation recovery experiment --- .../truncation-recovery-20261005/README.md | 40 +++ .../reports/truncation-recovery-20261005.md | 84 ++++++ .../tb21-truncation-recovery-20261005.json | 250 ++++++++++++++++++ docs/dev_notes/en/0.10.x.md | 51 ++++ docs/dev_notes/zh-CN/0.10.x.md | 39 +++ 5 files changed, 464 insertions(+) create mode 100644 benchmarks/harbor/configs/truncation-recovery-20261005/README.md create mode 100644 benchmarks/harbor/reports/truncation-recovery-20261005.md create mode 100644 benchmarks/harbor/results/tb21-truncation-recovery-20261005.json diff --git a/benchmarks/harbor/configs/truncation-recovery-20261005/README.md b/benchmarks/harbor/configs/truncation-recovery-20261005/README.md new file mode 100644 index 0000000..8cf7b36 --- /dev/null +++ b/benchmarks/harbor/configs/truncation-recovery-20261005/README.md @@ -0,0 +1,40 @@ +# Bounded truncation recovery experiment + +The [predeclared plan](experiment.json) selects the two failures from the +[v0.9.0 pilot20 baseline](../../reports/v0.9.0-baseline-20261005.md), with their +original revisions and budgets. Each task receives one fresh model run; no +score replaces a trial in the released baseline. + +Use a checkout of agent commit +`e1be6ed1f81a17d73677b414d4d913adb81e9e25` for the recorded behavior and wheel +version. The standard runner builds its current checkout, including local +source changes. Configure the official DeepSeek credentials as described in +the [adapter README](../../README.md), set +`ANTHROPIC_BASE_URL=https://api.deepseek.com/anthropic`, and leave +`ANTHROPIC_MODEL` unset so the configuration chooses `deepseek-flash`. + +Run dependency installation preflight first: + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter run \ + --config benchmarks/harbor/configs/truncation-recovery-20261005/install-only.json \ + --install-only +``` + +Both tasks lack a registered verifier dependency profile at this revision and +must explicitly report `not_configured`. After agent installation succeeds, +run the following two commands in separate terminals to reproduce the original +concurrent task pairing. Omit `--job-name` to generate fresh job names. + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter run \ + --config benchmarks/harbor/configs/truncation-recovery-20261005/model-extraction.json +uv run --project benchmarks/harbor python -m harbor_adapter run \ + --config benchmarks/harbor/configs/truncation-recovery-20261005/scheme.json +``` + +Record both task scores and actual recovery events. A task that passes without +reaching `max_tokens` does not demonstrate recovery effectiveness. The +[experiment report](../../reports/truncation-recovery-20261005.md) and +[structured results](../../results/tb21-truncation-recovery-20261005.json) +preserve that distinction. diff --git a/benchmarks/harbor/reports/truncation-recovery-20261005.md b/benchmarks/harbor/reports/truncation-recovery-20261005.md new file mode 100644 index 0000000..cd5df37 --- /dev/null +++ b/benchmarks/harbor/reports/truncation-recovery-20261005.md @@ -0,0 +1,84 @@ +# Bounded truncation recovery: two-task experiment + +The experimental agent scored **2/2 passed**, with **0 scored failures** and +**0 unscored trials** on the two tasks that failed in the frozen v0.9.0 +pilot20. Neither trial reached the generation cap, so neither exercised +automatic recovery. These fresh-run scores do not establish a benefit from the +policy. Deterministic tests cover the recovery branch and its safety/budget +boundaries; live effectiveness remains unproven. + +## Hypothesis and implementation + +A completed API response with `stop_reason="max_tokens"` previously ended the +headless run even when task time and replies remained. Allow one continuation +per headless run, using the original deadline, reply count, and per-reply token +limit. Keep partial visible text and a notice in request history; discard all +unexecuted tool calls and incomplete thinking from that response. Continue from +confirmed tool results and current files, without replaying prior tool actions. + +Record the continuation as `input.injected` with reason `truncation_recovery`, +while retaining each model attempt's stop reason, usage, and cost. A transport +retry or successful tool action does not renew the allowance. Another +truncation, the reply limit, or the original deadline ends the run. Interactive +mode continues to return to the user after a truncation. + +## Fixed inputs + +- Agent commit: `e1be6ed1f81a17d73677b414d4d913adb81e9e25`. +- Model: `deepseek/deepseek-flash` at `https://api.deepseek.com/anthropic`. +- 100 model replies and 65,536 generated tokens per reply. +- Model extraction: 720 seconds of work; Scheme: 2,220 seconds of work. +- Setup allowance: 1,800 seconds, unchanged from the frozen release baseline. +- One fresh run per task, no whole-trial retry. The two single-task standard + Harbor jobs ran together with at most two simultaneous trials. +- Official task revisions, images, package sources, and tests were preserved. + All 73 recorded task-file hashes remained unchanged, and task checksums + match preflight. Verifier timeouts alone may retry at the original per-attempt + limit, at most three attempts on the same agent output. + +The [predeclared plan](../configs/truncation-recovery-20261005/experiment.json) +contains the task revisions, budgets, source/config hashes, and analysis rules. +Both install-only checks passed with the experimental agent. Both verifier +dependency profiles explicitly report `not_configured`; installation success +is not verifier dependency certification. + +The preflight and both live-run wheels are byte-identical. Both task images and +full task checksums match the released baseline. Both official verifiers +completed on their first attempt; Scheme passed all 63 subcases. + +## Results + +| Task | Baseline reward | Experiment reward | Native outcome | Truncated responses | Recoveries | Executed tools after recovery | Agent seconds | Known estimated USD | +| --- | ---: | ---: | --- | ---: | ---: | ---: | ---: | ---: | +| `model-extraction-relu-logits` | 0 | 1 | `completed` | 0 | 0 | 0 | 361.7 | 0.054624 | +| `schemelike-metacircular-eval` | 0 | 1 | `completed` | 0 | 0 | 0 | 1398.5 | 0.180475 | + +Known estimated cost subtotal: **$0.235099**. +Usage accounting is **complete**; +cost accounting is **complete for reported tokens**. +There are 0 incomplete model attempts. Costs use the static +DeepSeek price table and are not provider billing totals. Agent seconds include +finalization overhead. + +The [structured results](../results/tb21-truncation-recovery-20261005.json) +include both trials, preflight, source/wheel hashes, recovery-event counts, +per-attempt verifier records, native outcomes, and usage. Raw journals, +trajectories, and test logs remain in the referenced Git-ignored `jobs/` paths. + +## Validation and interpretation + +- Core tests: 351 passed, 4 optional-transport skips with locked dependencies; + 355 passed with Anthropic SDK 1.5.0. +- Harbor workflow/ATIF contracts: 91 passed, including truncated tool calls + followed by a recovered model response. +- Deterministic tests exercise continuation through tool work to completion, + the once-per-run cap across transport retries, unchanged deadlines and reply + limits, discarded truncated tools/thinking, no duplicate committed tool + execution, unchanged interactive behavior, and retained usage/cost totals. +- Live trajectory validation and source/task integrity checks are recorded in + the structured results. + +The [released baseline](v0.9.0-baseline-20261005.md) remains **18/20**. These two +new results do not replace its failures or produce a revised pilot20 score. +They use fresh model generations and a different concurrent task pairing; +there is no deterministic counterfactual for the old failed responses. diff --git a/benchmarks/harbor/results/tb21-truncation-recovery-20261005.json b/benchmarks/harbor/results/tb21-truncation-recovery-20261005.json new file mode 100644 index 0000000..89fb944 --- /dev/null +++ b/benchmarks/harbor/results/tb21-truncation-recovery-20261005.json @@ -0,0 +1,250 @@ +{ + "schema_version": 1, + "generated_at": "2026-10-05T13:23:42.493492+00:00", + "status": "finished", + "agent_ref": "e1be6ed1f81a17d73677b414d4d913adb81e9e25", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "experiment_plan": "benchmarks/harbor/configs/truncation-recovery-20261005/experiment.json", + "started_at": "2026-10-05T12:56:30.236650+00:00", + "finished_at": "2026-10-05T13:23:21.319695+00:00", + "summary": { + "planned": 2, + "finished": 2, + "passed": 2, + "scored_failures": 0, + "unscored": 0, + "truncated_responses": 0, + "recovery_events": 0, + "passed_with_recovery": 0, + "known_estimated_cost_usd": "0.235099308", + "cost_complete": true, + "usage_complete": true, + "incomplete_model_attempts": 0 + }, + "jobs": [ + { + "job": "truncation-recovery-model-extraction-20261005", + "config": "benchmarks/harbor/configs/truncation-recovery-20261005/model-extraction.json", + "started_at": "2026-10-05T12:56:30.237330+00:00", + "exit_code": 0, + "finished_at": "2026-10-05T13:05:36.265597+00:00", + "wheel_sha256": "fc8e1ab0c3b66011f154e4d88d457227d695e26494b502a18dc7f0a64d1ce7f8", + "sources_match_plan": true + }, + { + "job": "truncation-recovery-scheme-20261005", + "config": "benchmarks/harbor/configs/truncation-recovery-20261005/scheme.json", + "started_at": "2026-10-05T12:56:30.237877+00:00", + "exit_code": 0, + "finished_at": "2026-10-05T13:23:21.318842+00:00", + "wheel_sha256": "fc8e1ab0c3b66011f154e4d88d457227d695e26494b502a18dc7f0a64d1ce7f8", + "sources_match_plan": true + } + ], + "preflight_trials": [ + { + "task": "terminal-bench/model-extraction-relu-logits", + "task_ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "task_checksum": "8b3048320aa04676089f9240563c2d7d381be03d8810bcf5f5d336cab538f523", + "agent_version": "0.9.1.dev3+ge1be6ed1f", + "status": "setup_passed", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + } + }, + { + "task": "terminal-bench/schemelike-metacircular-eval", + "task_ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6", + "task_checksum": "e4525988ae20585b6b90a2f1af2fcc9d18d6363d3cfcdd7be5305199428883d2", + "agent_version": "0.9.1.dev3+ge1be6ed1f", + "status": "setup_passed", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + } + } + ], + "unchanged_task_files": 73, + "trials": [ + { + "task": "terminal-bench/model-extraction-relu-logits", + "task_ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "task_checksum": "8b3048320aa04676089f9240563c2d7d381be03d8810bcf5f5d336cab538f523", + "trial": "model-extraction-relu-logits__6VEMYvp", + "job": "truncation-recovery-model-extraction-20261005", + "agent_version": "0.9.1.dev3+ge1be6ed1f", + "status": "scored", + "reward": 1.0, + "exception_type": null, + "result_path": "jobs/truncation-recovery-model-extraction-20261005/model-extraction-relu-logits__6VEMYvp/result.json", + "agent_setup_seconds": 133.286, + "agent_execution_seconds": 361.657, + "verifier_seconds": 25.871, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "budget_seconds": 720, + "verifier_attempts": { + "timeout_sec": 900.0, + "max_attempts": 3, + "attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 25.490705450996757 + } + ] + }, + "trajectory_path": "jobs/truncation-recovery-model-extraction-20261005/model-extraction-relu-logits__6VEMYvp/agent/trajectory.json", + "atif_valid": true, + "terminal": { + "status": "completed", + "duration_ms": 360007.277545, + "timestamp": "2026-10-05T13:04:52.194Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + }, + "metrics": { + "total_steps": 51, + "total_prompt_tokens": 713052, + "total_completion_tokens": 39047, + "total_cached_tokens": 701184, + "total_cost_usd": 0.054623904, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.054623904 + } + }, + "model_attempts": 25, + "truncated_responses": [], + "recovery_events": [], + "tools_executed_after_recovery": 0, + "incomplete_model_attempts": 0, + "known_cost_usd": 0.054623904, + "cost_complete": true, + "usage_complete": true, + "known_tokens": { + "prompt_tokens": 713052, + "completion_tokens": 39047, + "cached_tokens": 701184 + }, + "baseline": { + "reward": 0.0, + "terminal": { + "status": "completed", + "duration_ms": 299916.300286, + "timestamp": "2026-10-05T09:41:59.444Z", + "timestamp_source": "source_timestamp", + "outcome": "response_truncated" + }, + "agent_execution_seconds": 301.297, + "known_cost_usd": 0.079038732, + "cost_complete": true, + "usage_complete": true, + "trajectory_path": "jobs/v090-pilot20-720s-20261005/model-extraction-relu-logits__8mXWuta/agent/trajectory.json" + } + }, + { + "task": "terminal-bench/schemelike-metacircular-eval", + "task_ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6", + "task_checksum": "e4525988ae20585b6b90a2f1af2fcc9d18d6363d3cfcdd7be5305199428883d2", + "trial": "schemelike-metacircular-eval__UApkxB6", + "job": "truncation-recovery-scheme-20261005", + "agent_version": "0.9.1.dev3+ge1be6ed1f", + "status": "scored", + "reward": 1.0, + "exception_type": null, + "result_path": "jobs/truncation-recovery-scheme-20261005/schemelike-metacircular-eval__UApkxB6/result.json", + "agent_setup_seconds": 104.159, + "agent_execution_seconds": 1398.507, + "verifier_seconds": 84.589, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "budget_seconds": 2220, + "verifier_attempts": { + "timeout_sec": 2400.0, + "max_attempts": 3, + "attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 84.2060621099663 + } + ] + }, + "trajectory_path": "jobs/truncation-recovery-scheme-20261005/schemelike-metacircular-eval__UApkxB6/agent/trajectory.json", + "atif_valid": true, + "terminal": { + "status": "completed", + "duration_ms": 1397017.741137, + "timestamp": "2026-10-05T13:21:38.124Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + }, + "metrics": { + "total_steps": 121, + "total_prompt_tokens": 4948349, + "total_completion_tokens": 118411, + "total_cached_tokens": 4918784, + "total_cost_usd": 0.180475404, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.180475404 + } + }, + "model_attempts": 60, + "truncated_responses": [], + "recovery_events": [], + "tools_executed_after_recovery": 0, + "incomplete_model_attempts": 0, + "known_cost_usd": 0.180475404, + "cost_complete": true, + "usage_complete": true, + "known_tokens": { + "prompt_tokens": 4948349, + "completion_tokens": 118411, + "cached_tokens": 4918784 + }, + "baseline": { + "reward": 0.0, + "terminal": { + "status": "completed", + "duration_ms": 1161969.701524, + "timestamp": "2026-10-05T10:34:49.411Z", + "timestamp_source": "source_timestamp", + "outcome": "response_truncated" + }, + "agent_execution_seconds": 1163.351, + "known_cost_usd": 0.298166292, + "cost_complete": false, + "usage_complete": false, + "trajectory_path": "jobs/v090-pilot20-2220s-20261005/schemelike-metacircular-eval__RRuG7fV/agent/trajectory.json" + } + } + ], + "accounting_note": "Costs are token-based estimates using the static price table, not provider bills. Incomplete attempts retain unknown usage/cost, not zero.", + "comparison_note": "One fresh trial per task, selected because both failed in the released baseline. The two standard jobs run together; this is not a controlled replay or a full pilot score. A pass without truncation does not exercise the recovery policy.", + "artifact_checks": { + "task_images": [ + { + "image": "alexgshaw/model-extraction-relu-logits:20251031", + "image_id": "sha256:52fe1f089f38650f0dc22d7531ee6e01ebd526de1b349aaa2ecb331dca9fabce", + "matches_baseline": true + }, + { + "image": "alexgshaw/schemelike-metacircular-eval:20251031", + "image_id": "sha256:b485be38bf80d56c1a4cd9da95bbf7ec129dc03f6786b86c0525518a96f6d1db", + "matches_baseline": true + } + ], + "preflight_and_run_wheel_sha256": "fc8e1ab0c3b66011f154e4d88d457227d695e26494b502a18dc7f0a64d1ce7f8", + "task_checksums_match_release_baseline": true + } +} diff --git a/docs/dev_notes/en/0.10.x.md b/docs/dev_notes/en/0.10.x.md index df4e4ea..ccad9ff 100644 --- a/docs/dev_notes/en/0.10.x.md +++ b/docs/dev_notes/en/0.10.x.md @@ -47,3 +47,54 @@ investigating bounded truncation recovery in a separate experiment while preserving this run's scores. Pilot20 is a small trial cohort, not the full 89-task score. Historical runs with different dates and conditions do not isolate the causal effect of a single change. + +### Experiment: continue once after truncation within the original budgets + +The first experiment changes only headless handling of `max_tokens`: allow one +automatic recovery per run, keeping the original deadline, reply count, and +per-reply token limit. The truncated response still consumes budget and retains +its usage, cost, and original log. Skip all its tool calls and remove incomplete +thinking from follow-up requests. The recovery prompt asks the model to +continue from confirmed tool results and current files without replaying +previously executed tools. A second truncation still stops the run; interactive +mode still waits for user input. + +Record the recovery prompt using schema v4's `input.injected` with reason +`truncation_recovery`, distinguishing a truncated response from an actual +recovery attempt in Journal and ATIF. Normal tool execution and transport +retries do not renew the allowance. + +On 2026-10-05, pin commit `e1be6ed1f81a17d73677b414d4d913adb81e9e25` and run each +of the two previously failed tasks once from scratch. Keep the same model, +100 replies, 65,536 tokens, 720/2,220-second work budgets, 1,800-second setup +allowance, and native per-attempt verifier limits. Both installation preflights +pass, with both verifier dependency profiles explicitly `not_configured`. +Preflight and live-run wheels are byte-identical. Task images and full task +checksums match the released baseline. + +| Task | This run | Truncated responses | Recoveries | +| --- | --- | ---: | ---: | +| `model-extraction-relu-logits` | Passed; about 362 seconds, 25 model replies | 0 | 0 | +| `schemelike-metacircular-eval` | Passed, 63/63 subcases; about 1,399 seconds, 60 model replies | 0 | 0 | + +Both official verifiers complete on their first attempt; there are no unscored +trials. Token records are complete, with an estimated cost of **$0.235099**, +which remains distinct from the provider bill. **Neither task triggered +recovery, so these passes cannot be attributed to the new policy, and the +original 18/20 must not be rewritten as 20/20.** These are two fresh samples +with different model generations and concurrent task pairing, without a +deterministic replay of the original failed responses as a control. + +Deterministic tests cover recovery through tool work to completion, stopping +on a second truncation, avoiding duplicate injection during transport retries, +not replaying completed tools or executing truncated tools, and enforcing the +deadline, final reply, and usage/cost accounting. Core tests have 351 passes +and 4 optional-transport skips with locked dependencies, and 355 passes with +SDK 1.5.0. Harbor compatibility and workflow checks have 91 passes. Tests cover +the mechanism's correctness; this live experiment does not establish its +benefit on tasks that actually truncate. + +Keep the [experiment report](../../../benchmarks/harbor/reports/truncation-recovery-20261005.md), +[structured results](../../../benchmarks/harbor/results/tb21-truncation-recovery-20261005.json), +and [reproduction configuration](../../../benchmarks/harbor/configs/truncation-recovery-20261005/README.md). +Do not keep rerunning until a favorable sample appears. diff --git a/docs/dev_notes/zh-CN/0.10.x.md b/docs/dev_notes/zh-CN/0.10.x.md index 3326a43..1051a26 100644 --- a/docs/dev_notes/zh-CN/0.10.x.md +++ b/docs/dev_notes/zh-CN/0.10.x.md @@ -37,3 +37,42 @@ trajectory 均通过 ATIF 校验，保留所有试次，没有用重跑替换失 原始日志留在 Git 忽略的 `jobs/` 中。后续优先研究有界的截断恢复策略，作为独立 实验评估；本轮分数保持不变。20 题只是小规模试跑，不能代表完整 89 题的分数， 也不能从不同日期、不同运行条件的历史结果推导单一改动的因果效果。 + +### 实验：一次截断后允许在原预算内继续 + +第一项实验只改变 headless run 对 `max_tokens` 的处理：每次 run 最多自动恢复 +一次，沿用原始截止时间、回合计数和单次 token 上限。截断回复仍计入预算并保留 +usage、费用和原始日志；它的工具调用全部跳过，不完整的 thinking 不进入后续 +请求。恢复提示要求模型从已确认的工具结果和当前文件继续，不重放先前已执行的 +工具。第二次截断仍停止，交互模式仍等待用户输入。 + +恢复提示沿用 schema v4 的 `input.injected`，reason 为 `truncation_recovery`， +可在 Journal 和 ATIF 中区分“发生过截断”与“实际启动过恢复”。正常工具执行或 +传输重试不会重置恢复额度。 + +于 2026-10-05 固定提交 `e1be6ed1f81a17d73677b414d4d913adb81e9e25`，对原来两道 +失败题各做一次全新运行。模型、100 回合、65,536 token、720/2,220 秒工作预算、 +1,800 秒安装时限及官方逐次验证时限均保持不变。两题安装预检通过，verifier +依赖 profile 均明确为 `not_configured`。预检与正式运行的 wheel 完全一致， +任务镜像与完整任务 checksum 均与发布基线一致。 + +| 题目 | 本轮结果 | 截断次数 | 恢复次数 | +| --- | --- | ---: | ---: | +| `model-extraction-relu-logits` | 通过；约 362 秒，25 次模型回复 | 0 | 0 | +| `schemelike-metacircular-eval` | 通过，63/63 子例；约 1,399 秒，60 次模型回复 | 0 | 0 | + +两题官方 verifier 均第一次完成，0 题未评分。token 记录完整，费用估算合计 +**$0.235099**，仍不是 provider 实际账单。**两题都没有触发恢复，因此不能把 +这次通过归因于新策略，也不能把原来的 18/20 改写成 20/20。** 本轮是不同模型 +生成与不同并发配对下的两次新样本，没有重放原失败响应的确定性对照。 + +确定性测试覆盖了恢复后继续执行工具直至完成、第二次截断停止、传输重试期间 +不重复注入、不重放已完成工具、不执行截断工具，以及截止时间、最后回合和 +usage/费用累计。锁定依赖时核心测试 351 通过、4 项可选传输测试跳过；SDK 1.5.0 +下 355 项通过；Harbor 兼容性与流程检查 91 项通过。机制的正确性已有测试覆盖， +真实截断任务上的收益仍未得到本轮实验证明。 + +保留[实验报告](../../../benchmarks/harbor/reports/truncation-recovery-20261005.md)、 +[结构化结果](../../../benchmarks/harbor/results/tb21-truncation-recovery-20261005.json) +与[复跑配置](../../../benchmarks/harbor/configs/truncation-recovery-20261005/README.md)， +不继续重跑直到获得有利样本。 From 382cb38413d7d5368ad1be97e85142f280172a5b Mon Sep 17 00:00:00 2001 From: minixalpha Date: Mon, 5 Oct 2026 07:25:52 -0700 Subject: [PATCH 3/4] test(benchmark): predeclare three recovery trials per task --- .../README.md | 41 ++++ .../experiment.json | 186 ++++++++++++++++++ .../install-only.json | 27 +++ .../model-extraction.json | 22 +++ .../scheme.json | 22 +++ 5 files changed, 298 insertions(+) create mode 100644 benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/README.md create mode 100644 benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/experiment.json create mode 100644 benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/install-only.json create mode 100644 benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/model-extraction.json create mode 100644 benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/scheme.json diff --git a/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/README.md b/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/README.md new file mode 100644 index 0000000..4ea6087 --- /dev/null +++ b/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/README.md @@ -0,0 +1,41 @@ +# Three additional runs per task + +This [predeclared plan](experiment.json) follows the earlier +[two-task experiment](../../reports/truncation-recovery-20261005.md), which +passed both tasks without exercising truncation recovery. Run each task three +more times from scratch, preserving all six results. These are additional +samples, not replacements for earlier scores. + +Keep the same agent/adapter Python source hashes, task revisions, images, model, +endpoint, token cap, reply cap, and work/setup/verifier budgets as the preceding +experiment. Runtime behavior is from commit +`e1be6ed1f81a17d73677b414d4d913adb81e9e25`; the runner records the exact checkout +commit and built wheel for this round. + +From the repository root, configure official DeepSeek credentials, set +`ANTHROPIC_BASE_URL=https://api.deepseek.com/anthropic`, and leave +`ANTHROPIC_MODEL` unset. Run installation preflight first: + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter run \ + --config benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/install-only.json \ + --install-only +``` + +Both verifier dependency profiles must explicitly report `not_configured`. +After installation succeeds, run the six entries in `job_queue`, taking each +configuration path relative to this directory. Limit concurrent jobs to two: +each configuration contains one task and one attempt. For example: + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter run \ + --config benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/scheme.json \ + --job-name truncation-recovery-repeat3-scheme-1-20261005 +``` + +Use fresh job names when reproducing the experiment; the runner never overwrites +existing results. No whole-trial retries or replacements are allowed. Only +verifier timeouts may retry, at most three attempts with the original per-attempt +limit and the same agent output. Record truncation, actual recovery, subsequent +tool execution, terminal outcome, score, and accounting completeness per trial. +A passing trial without truncation does not demonstrate recovery effectiveness. diff --git a/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/experiment.json b/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/experiment.json new file mode 100644 index 0000000..2de4fa4 --- /dev/null +++ b/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/experiment.json @@ -0,0 +1,186 @@ +{ + "created_at": "2026-10-05T14:25:46.533541+00:00", + "question": "Across three additional fresh runs of each previously failed task, does natural truncation recur, and can bounded recovery continue useful work and reach a passing score?", + "baseline_results": "benchmarks/harbor/results/tb21-v090-baseline-20261005.json", + "baseline_agent_ref": "c0605369d277352ac0d5ec20d02017b7cb149314", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "change": "One automatic truncation recovery per headless run. Keep the token cap and original deadline and turn count; skip truncated tools and incomplete thinking.", + "max_turns": 100, + "max_tokens": 65536, + "agent_setup_timeout_seconds": 1800, + "attempts_per_task": 3, + "whole_trial_retries": 0, + "execution": "Preflight both tasks, then schedule six independent single-task standard jobs with at most two running at once. Start jobs in the declared queue order whenever a slot is free. Each job uses n_attempts=1; preserve all six results regardless of scores or truncation.", + "verifier_policy": "Original per-attempt limit; only verifier timeouts may retry, at most three attempts on the same agent output.", + "preflight": "Both task profiles are unsupported and must be recorded as not_configured; installation success does not certify verifier dependencies.", + "analysis": "Report all six additional runs separately from the earlier two-run experiment and released baseline. Count completed max_tokens responses, actual truncation_recovery injections, subsequent model calls and executed tools, second truncations, remaining budgets, and official verifier rewards. Distinguish continuation from successful task recovery. A pass without truncation does not exercise recovery. Passing after recovery provides a live recovery example but is not a controlled counterfactual. Keep setup/verifier errors separate from scored failures; no replacement runs or additional trials after seeing results.", + "tasks": [ + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "native_agent_timeout_seconds": 900, + "work_budget_seconds": 720, + "verifier_timeout_seconds": 900.0, + "docker_image": "alexgshaw/model-extraction-relu-logits:20251031", + "docker_image_id": "sha256:52fe1f089f38650f0dc22d7531ee6e01ebd526de1b349aaa2ecb331dca9fabce", + "docker_repo_digests": [ + "alexgshaw/model-extraction-relu-logits@sha256:52fe1f089f38650f0dc22d7531ee6e01ebd526de1b349aaa2ecb331dca9fabce" + ], + "task_file_sha256": { + "instruction.md": "c9e179dd099065a34c362a865b0eb9fab5f7d96be582cf117e7519580c036ebe", + "task.toml": "eb64dfaf8ae88868911ae28855cdc4f767de464001e2704919015b595be6dc29", + "tests/test.sh": "3b2d8f5e0353c2bab3515d6387468a863a4f9f241ee131c73f545f3fdf9e2d52", + "tests/test_outputs.py": "e5189c7eb9942ede55e1b48a80388590b80a486b75497613f257b3373ad9b2c9" + } + }, + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6", + "native_agent_timeout_seconds": 2400, + "work_budget_seconds": 2220, + "verifier_timeout_seconds": 2400.0, + "docker_image": "alexgshaw/schemelike-metacircular-eval:20251031", + "docker_image_id": "sha256:b485be38bf80d56c1a4cd9da95bbf7ec129dc03f6786b86c0525518a96f6d1db", + "docker_repo_digests": [ + "alexgshaw/schemelike-metacircular-eval@sha256:b485be38bf80d56c1a4cd9da95bbf7ec129dc03f6786b86c0525518a96f6d1db" + ], + "task_file_sha256": { + "instruction.md": "3af5f9567b62f93683db7518ec6fcb609f83796258aa02166b5a7091c5b69031", + "task.toml": "8dae8eb226183b0ca92331c1106152a7abc93b03b4e5a95291b86a4ddc9b44e0", + "tests/interp.py": "efb8b5cdcf78b73afcf39d59600a1f355ca89b7bac9854bb01f110cc77bea2c1", + "tests/shadow_test/01-factorial.scm": "540e81fac476f04dc510ba107721d606ac980760088b2136889e7d4bf884e5c5", + "tests/shadow_test/02-fibonacci.scm": "628957171e0c4a3ee373635923fcd29e75bdf4bf13a402ac26fe94a5fc1443ac", + "tests/shadow_test/03-list-operations.scm": "fe8f17f59fe4045582fce363d52df95ba7c4de7345542c8bd4993a9f2f396235", + "tests/shadow_test/04-higher-order.scm": "9c2eba8c91bb7b224de56d789207d2eaad98bbe1f8c360d6eca4f014f42f9230", + "tests/shadow_test/05-simple-io.scm": "1eba30db30824fcf6048ea8e49b84e85c866cefce30862fce49b300e4416843c", + "tests/shadow_test/06-interactive-io.scm": "4fbc0e2d6e42c5a9495e549ccb13c02fcaeddf8fdaf8d2cf5de833116f12610a", + "tests/shadow_test/08-progn-sequencing.scm": "3bf8236dc5356d29ab8dbfe1b0f4c519354fbb23c8f3eaf946779c28b80feea3", + "tests/shadow_test/09-mutual-recursion.scm": "3bfa1f77e602c7039f8c8c70315e6d4fff6eed737dab0f0398e5bdf690137c2c", + "tests/shadow_test/10-advanced-features.scm": "b69ef497088fcf0aec388789a17561b783b8cdd5915719915a51c5ba189c31ff", + "tests/shadow_test/accumulator_patterns.scm": "8b9192796a6f9c3155cc5e44d0a0ba6a46d20a0d94e265b69b23ec8af43e4a07", + "tests/shadow_test/binary_tree.scm": "5a3dafc8844969cebb5809f253f4e54de7df480f378da8197f870badef03ee47", + "tests/shadow_test/church_numerals.scm": "31d075785f2f605bac3bd9ac2d99fcf7eaaed6686eb9afeab1d40bf5f35bbf2c", + "tests/shadow_test/closures.scm": "c7d025a376987f5ee22f529847b66eaf53f0af7033acbb4055cedb734168d612", + "tests/shadow_test/continuation_passing.scm": "79e7445c42227846f1adfaf08c17e278aac7a34df77482900b068be4fd248080", + "tests/shadow_test/currying.scm": "690a8cccbb1d249c8f6fcabc0193cecc5d0e4c1af39a0bb67c7e95fcff8e393e", + "tests/shadow_test/filter_operations.scm": "485a0563b81d002e368c5e4b2b7cc98a5306163701053630f416e3b783a2a238", + "tests/shadow_test/fold_operations.scm": "461b33db6b3f5580122d4c6dff0fd6d11c7f044865ffd684ade67d9b296a5a24", + "tests/shadow_test/function_composition.scm": "65ceee00188b12c3549134449e2f938d010c2c97dba090f5b53244abb3579b1f", + "tests/shadow_test/lazy_evaluation.scm": "edd95748de14aadf916c65209b5bcb8759d3e33b7ec935289a5128a656d33bec", + "tests/shadow_test/list_operations.scm": "660963851ecb49639f66557bd88db1e73db323186742145f33cb2e14d3af1103", + "tests/shadow_test/map_operations.scm": "974ac42d892f4b62b9b21ef7844581624dad811b38d35944f1005cc47f249856", + "tests/shadow_test/memoization.scm": "25482c1912fab1235651b1b7acd727f2ea6a55dc5c1919f841b44cfc32107b05", + "tests/shadow_test/mutual_recursion.scm": "2fbc9562e863cb2b325f86419fb2a7588265aaf5576295550722ee4274663712", + "tests/shadow_test/nested_defines.scm": "23ad3b2f936949501466da784511ea2e6215e2b198ca371bef86f6af877ca923", + "tests/shadow_test/oeis_sequences.scm": "1b586df6af2429a54808f7587f60021ba2b2c9bcc2ef2f9cbe44339699853c62", + "tests/shadow_test/oeis_sequences2.scm": "300da0b0dc20b883e01142adea9eb6da1a583a1b969ac0be47aa32c1ab66340e", + "tests/shadow_test/oeis_sequences3.scm": "dea198e874bf518b93c8406a843b64c1586cf4ce780fec8c444f8343e9322c75", + "tests/shadow_test/recursive_structures.scm": "64f1a430bf04a627f5e27bf004805544b60406f8a123d186516774dfcf3846cd", + "tests/shadow_test/test_read.scm": "fda448c22685e2f9af37ff7b986ee5e906d485a2db16f62eec113149f866870f", + "tests/shadow_test/variadic_functions.scm": "ac940ac0273e87ee06280fee192194ab5cdc14c88b61da9cfba92105e66b8f2f", + "tests/shadow_test/y_combinator.scm": "38d25b02d54342ab5610ad03fcef68c5f1d4eef8902295645f6acceb853c9860", + "tests/test/01-factorial.scm": "2623000a742c63d50ae1851854826247caee129c7d1613be6c7131d21f26a6ad", + "tests/test/02-fibonacci.scm": "db769cef446d5da9e9b1abccdf9ca644e239938095a98c21551aee25255a57d4", + "tests/test/03-list-operations.scm": "1a14b4100dedef30010fed62ea9fd295add1b0facb2e8e6e27c8d8eb3d90111b", + "tests/test/04-higher-order.scm": "59cd17bbffad641a37729b17c5bf9178b8bae7289fe89cc7135eb0bfec2b4fd6", + "tests/test/05-simple-io.scm": "3deeb3275df1b771449ebaebfcb9a934f375415e138c5e9f3685516c69a1f7a0", + "tests/test/06-interactive-io.scm": "aa3cae79b79dca90ac9d61a1babf64d6a5bb6559798c7727c11be02d8ddfd216", + "tests/test/08-progn-sequencing.scm": "4c0a4d980cf4223d1d17715d58adbe27dc120eced257a0d9f42de952c094bc5b", + "tests/test/09-mutual-recursion.scm": "939e18f4bd25b5ab0e2a4a767bf7eb6b0d8fb3530d28bf236f06cf27f2d90ee8", + "tests/test/10-advanced-features.scm": "c4cc5fb412df363875768c6a1e645255f6bc8f9ddf988d1ec2f4a266cedb8ec7", + "tests/test/README.md": "1f2174c7f12721f9f1898cda190c2886fb3ef9d35fd085f315da5814657104ae", + "tests/test/accumulator_patterns.scm": "50d246e00577d1f3ab3f0462c2b9618ea63680c33fcca12369cd1707b010cb5f", + "tests/test/binary_tree.scm": "c95f59ccac1ea4a9c846afddc993aa40ac081c0f9e6731544530109b269b67ae", + "tests/test/calculator.scm": "a09213ec8a2ec52486b9346c420ddb7c537576ef71b1cc5c6ec61d13d9139f61", + "tests/test/church_numerals.scm": "f37d858f32a9c5ba8868d094018c92516acadf0ea4e55b7c36f14320cb0c69a6", + "tests/test/closures.scm": "3ecfc35289479288876e20bf7f55e5b8eda84acd35937e89e3d062308a99aeff", + "tests/test/continuation_passing.scm": "01049581e582f7d35ef40ef68e1827d012da619beea57c1c74675a5f245318d7", + "tests/test/currying.scm": "2844d54eee9a94af05a11b4aae06d6e7823d193f17235c8f90bf98926cbca75d", + "tests/test/filter_operations.scm": "f6bc75f6da9ece96c0616e3570e4df59d0f2eb4655375d3a46b0349c86cc5dd8", + "tests/test/fold_operations.scm": "05aab492ad914baecd19cf454ed587e7a031e94b8990500eb1c869c15da6fc0b", + "tests/test/function_composition.scm": "9b56dc78823e7d35adb213744c0f8e95684074d6a12bb98c8c05755788be5b06", + "tests/test/lazy_evaluation.scm": "ea5912a53a35b65bf2bee9483dca90814b8baf52ae3ae50c2d4589f794dca54d", + "tests/test/list_operations.scm": "d7e70621afbbb787cf646e2089bccfbd3681fbb2e70325f3e50f56ccd30dbd9d", + "tests/test/map_operations.scm": "0cebddb3d4a50e4b6f6bfc0e8c12762d3f42609030aef077d8ab4bedda4718d9", + "tests/test/memoization.scm": "cf8202df22407dbba72ca7f660d8ab7cd4ca962b215c36cc491d5c1a474c160b", + "tests/test/mutual_recursion.scm": "a3dc23a96ec1cf05b0be4a4f0b6d3da9e2bad1aa3f7a83c7566f71f980a67fe2", + "tests/test/nested_defines.scm": "21faa0ccaf744cb8c9d283bbfd0c36e195c803c85f511e70a99267c0a85b6216", + "tests/test/oeis_sequences.scm": "d1f77161b367176b3dfab55a34466e511c602c7ae359e05f5a715627b93dca6a", + "tests/test/oeis_sequences2.scm": "5679ef0f99743114d8c06edcdcde1fea9474b71aeb57c2069f1ee33df1970bce", + "tests/test/oeis_sequences3.scm": "5dc8b44158b2648a6e9766f6bcdf21553a07bf487d79e64ebb418511ef3b8186", + "tests/test/recursive_structures.scm": "67838ff5bac26ea359139720adec1d5700176147b0062065c7def9715e8a2aeb", + "tests/test/test_read.scm": "5b9ee381c3cf45b2fc8c2e7b490857fb67f1ac78653b586d7f2848daf6853809", + "tests/test/variadic_functions.scm": "c9b847c50840c6977904e5033a07f6108cc3a54012b02d916e79025fb43e0f6e", + "tests/test/y_combinator.scm": "971144dcb2b04aaa3b34d9279459bd7d2e8256c585d17809465848c2589d36d8", + "tests/test.sh": "4770437ea96c3cc84684b4f99d55fb148fcac09f9ea1e8ef49de487716e6c334", + "tests/test_outputs.py": "fbb84e3c0c2f72ed7de90f54380f860052c16155e7d0c9076cd57c966e9be7ed" + } + } + ], + "source_sha256": { + "src/nanopycodeagent/__init__.py": "b922299f65b3db2fb1514df5aeda4413e599f595d2595f45a353b4dc551a1eb1", + "src/nanopycodeagent/agent.py": "8e28a5c3fcd140441484c46d1e62ec811d325aaff484141a4b9837baead6fc52", + "src/nanopycodeagent/atif.py": "ca4793b971d125d904888b858452db8db30e21f0d9b87818133062fb6cebe270", + "src/nanopycodeagent/bash_tool.py": "dcf0366fb9ebdca258d4c40b1a50b78b2ec79fc4b85cbb253ac9a9ce9534574f", + "src/nanopycodeagent/cli.py": "e335789b3f09440abd441547a2e3e2feb32b90c68e9ba5fc8237de8f6bac2932", + "src/nanopycodeagent/cost.py": "30a8df17942c424796e8532f45e1cb0784ed6541cea44284b863db3dae2f7b70", + "src/nanopycodeagent/deadline.py": "26a861721e6897a4e02e6ce364cb3ef1f1dc7d4ad9fb2f772404276f5c9e9d5f", + "src/nanopycodeagent/edit_tool.py": "78e2bec911279082b081366ab05cb4cbb37d8910b20b6245cccd4b0424489d0d", + "src/nanopycodeagent/event_journal.py": "8974919c2d460afb52bc57ec37e7e271851290c0845d01b4ad0f9f760cde0814", + "src/nanopycodeagent/read_tool.py": "e0075da19f2e1c6ccf34a3056a0aec92398e5afe67869937a096c6f1e04181c3", + "src/nanopycodeagent/settings.py": "491bf5c6eca50eb4c5fe04b7bfe62a8bfc7d2e1fad05b8eeac1680c2fcf3674e", + "src/nanopycodeagent/terminal.py": "8d27b40976b2b6d476e5106dfea982b055115435b4ba16b7f758412a05b75f04", + "src/nanopycodeagent/tool_validation.py": "3a1a85a7222dd1504bdf58f37e65a51d11de13d16c4e7df42c72b4cd58b5e5e5", + "src/nanopycodeagent/transport.py": "152d3ff78be0e3b2d92e0abbbfb652d73466aff57ad316c06dc0e072411167fe", + "src/nanopycodeagent/write_tool.py": "7671e83464fa1e942f25742fc573d0712e32053b5fc0291257884b027f7f6206", + "benchmarks/harbor/src/harbor_adapter/__init__.py": "513ad49c721b3e73293095c6a0486a791483e9aa2b51e0e9ad4b7816af07e76b", + "benchmarks/harbor/src/harbor_adapter/__main__.py": "db89d582fba45aad72147317f595531af49c434e3b23c65f22d7491f0b10551d", + "benchmarks/harbor/src/harbor_adapter/_verifier_runner.py": "2eb73c3e76d7360f2fa16018f0e933797bf982e589236095b5f98750f3807a7c", + "benchmarks/harbor/src/harbor_adapter/adapter.py": "d1aa154ec6972d706f0725cfa69c56635506d4499dff2a129e2c957ea7034c02", + "benchmarks/harbor/src/harbor_adapter/bootstrap.py": "f499bfddfc3d29338cae15a7f8f6e9c66d19eb69ad9577e607ced4b1aa057be2", + "benchmarks/harbor/src/harbor_adapter/verifier.py": "ec26fdefd69be1c5c450c9ba69d749d4a9d320a2a170b8db34421f0573d9b493" + }, + "configuration_sha256": { + "model-extraction.json": "f8baf52e182e6b6ab669644c904c1c961db5e6083c6c18b8d1479932a9b26588", + "scheme.json": "0bf57205bb0d82ea0342483a269eb995441a7c427fec40f0678f3a69b1a87b0b", + "install-only.json": "75e6b3f9d567c1c2db953deef82cc43ec8ea31c3c79ab15fa11599c68531aaaf" + }, + "behavior_commit": "e1be6ed1f81a17d73677b414d4d913adb81e9e25", + "previous_experiment_results": "benchmarks/harbor/results/tb21-truncation-recovery-20261005.json", + "planned_trials": 6, + "preflight_job": "truncation-recovery-repeat3-install-20261005", + "job_queue": [ + { + "job": "truncation-recovery-repeat3-scheme-1-20261005", + "config": "scheme.json", + "attempt": 1 + }, + { + "job": "truncation-recovery-repeat3-model-extraction-1-20261005", + "config": "model-extraction.json", + "attempt": 1 + }, + { + "job": "truncation-recovery-repeat3-model-extraction-2-20261005", + "config": "model-extraction.json", + "attempt": 2 + }, + { + "job": "truncation-recovery-repeat3-scheme-2-20261005", + "config": "scheme.json", + "attempt": 2 + }, + { + "job": "truncation-recovery-repeat3-model-extraction-3-20261005", + "config": "model-extraction.json", + "attempt": 3 + }, + { + "job": "truncation-recovery-repeat3-scheme-3-20261005", + "config": "scheme.json", + "attempt": 3 + } + ] +} diff --git a/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/install-only.json b/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/install-only.json new file mode 100644 index 0000000..2c71e92 --- /dev/null +++ b/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/install-only.json @@ -0,0 +1,27 @@ +{ + "n_concurrent_trials": 2, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17" + }, + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6" + } + ], + "install_only": true +} diff --git a/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/model-extraction.json b/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/model-extraction.json new file mode 100644 index 0000000..044f999 --- /dev/null +++ b/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/model-extraction.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 2, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/model-extraction-relu-logits", + "ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17" + } + ] +} diff --git a/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/scheme.json b/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/scheme.json new file mode 100644 index 0000000..0c80195 --- /dev/null +++ b/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/scheme.json @@ -0,0 +1,22 @@ +{ + "n_concurrent_trials": 2, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "override_setup_timeout_sec": 1800, + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 2220 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/schemelike-metacircular-eval", + "ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6" + } + ] +} From 903844ac160a3af2cc89fcb2fea5a7bbf7c4218c Mon Sep 17 00:00:00 2001 From: minixalpha Date: Mon, 5 Oct 2026 08:26:14 -0700 Subject: [PATCH 4/4] docs(benchmark): record six truncation recovery trials --- .../README.md | 4 + .../reports/truncation-recovery-20261005.md | 7 + .../truncation-recovery-repeat3-20261005.md | 117 +++ ...-truncation-recovery-repeat3-20261005.json | 880 ++++++++++++++++++ docs/dev_notes/en/0.10.x.md | 59 ++ docs/dev_notes/zh-CN/0.10.x.md | 43 + 6 files changed, 1110 insertions(+) create mode 100644 benchmarks/harbor/reports/truncation-recovery-repeat3-20261005.md create mode 100644 benchmarks/harbor/results/tb21-truncation-recovery-repeat3-20261005.json diff --git a/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/README.md b/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/README.md index 4ea6087..0ebaa8e 100644 --- a/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/README.md +++ b/benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/README.md @@ -39,3 +39,7 @@ verifier timeouts may retry, at most three attempts with the original per-attemp limit and the same agent output. Record truncation, actual recovery, subsequent tool execution, terminal outcome, score, and accounting completeness per trial. A passing trial without truncation does not demonstrate recovery effectiveness. + +The recorded run used checkout `382cb38413d7d5368ad1be97e85142f280172a5b`. +See the [six-run report](../../reports/truncation-recovery-repeat3-20261005.md) +and [structured results](../../results/tb21-truncation-recovery-repeat3-20261005.json). diff --git a/benchmarks/harbor/reports/truncation-recovery-20261005.md b/benchmarks/harbor/reports/truncation-recovery-20261005.md index cd5df37..8fc4a7b 100644 --- a/benchmarks/harbor/reports/truncation-recovery-20261005.md +++ b/benchmarks/harbor/reports/truncation-recovery-20261005.md @@ -82,3 +82,10 @@ The [released baseline](v0.9.0-baseline-20261005.md) remains **18/20**. These tw new results do not replace its failures or produce a revised pilot20 score. They use fresh model generations and a different concurrent task pairing; there is no deterministic counterfactual for the old failed responses. + +## Follow-up + +The maintainer subsequently requested a fixed set of three additional runs per +task. The [separate six-run report](truncation-recovery-repeat3-20261005.md) +records 4/6 passes and one natural truncation followed by useful continuation, +but no passing score after recovery. The original results above remain unchanged. diff --git a/benchmarks/harbor/reports/truncation-recovery-repeat3-20261005.md b/benchmarks/harbor/reports/truncation-recovery-repeat3-20261005.md new file mode 100644 index 0000000..d1d2f42 --- /dev/null +++ b/benchmarks/harbor/reports/truncation-recovery-repeat3-20261005.md @@ -0,0 +1,117 @@ +# Truncation recovery: three additional runs per task + +The six predeclared fresh runs scored **4/6 passed**, **2 scored failures**, and +**0 unscored trials**. Model extraction passed **1/3**; Scheme passed **3/3**. +One model-extraction run naturally truncated on its second response, reproducing +the released baseline's failure pattern. Automatic recovery continued useful +work, but that trial still failed the official test. The experiment demonstrates +live continuation after truncation, not successful recovery of a passing score. + +## Fixed experiment + +The maintainer requested three additional runs of each previously failed task +after the [initial two-run experiment](truncation-recovery-20261005.md) passed +without any truncation. The [plan](../configs/truncation-recovery-repeat3-20261005/experiment.json) +was committed before model work, with all six jobs and their queue order fixed. +Every job ran once in a fresh container; no outcome was replaced or retried. + +- Checkout: `382cb38413d7d5368ad1be97e85142f280172a5b`. +- Wheel version: `0.9.1.dev5+g382cb3841`. +- All 21 agent/adapter Python source hashes match the earlier experiment at + `e1be6ed1f81a17d73677b414d4d913adb81e9e25`. This round changes no runtime code. +- Model: `deepseek/deepseek-flash`, official + `https://api.deepseek.com/anthropic` endpoint. +- 100 model replies, 65,536 output tokens per reply; 720 seconds of work for + model extraction and 2,220 seconds for Scheme; 1,800 seconds for setup. +- Two simultaneous single-task jobs at most, using the repository's standard + Harbor entry point and verified dependency cache. +- Original task revisions, images, package sources, and tests. Verifier limits + remain 900/2,400 seconds per attempt; only timeouts may retry, at most three + attempts on the same output. All six verifiers completed on their first attempt. + +Both install-only trials passed. Both verifier dependency profiles explicitly +report `not_configured`; these are installation successes, not certification of +verifier dependencies. The preflight and all six run wheels are byte-identical, +with SHA256 `9f519124b62314d5b3625004441e0bfc29473533f29cef2dcf3640e55094800f`. +Both full task checksums and image IDs match the released baseline. All 73 +recorded task-file hashes remained unchanged. All six native ATIF trajectories +validate, and all six native terminal outcomes are `completed`. + +The six jobs ran from **2026-10-05 14:27:47 to 15:20:14 UTC**, including setup, +model work, verification, and cleanup. The [reproduction instructions](../configs/truncation-recovery-repeat3-20261005/README.md) +record the fixed schedule and standard commands. + +## All six results + +| Task | Run | Reward | Truncations | Recoveries | Model replies | Largest reply tokens | Agent seconds | Official result | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| Model extraction | 1 | 0 | 0 | 0 | 27 | 17,791 | 414.4 | Missing rows 2, 27 | +| Model extraction | 2 | 0 | 1 | 1 | 17 | 65,536 | 554.1 | Missing rows 24 | +| Model extraction | 3 | 1 | 0 | 0 | 23 | 53,538 | 483.9 | Passed | +| Scheme | 1 | 1 | 0 | 0 | 41 | 17,495 | 1080.7 | 63/63 subcases | +| Scheme | 2 | 1 | 0 | 0 | 79 | 35,869 | 1281.1 | 63/63 subcases | +| Scheme | 3 | 1 | 0 | 0 | 60 | 23,002 | 829.9 | 63/63 subcases | + +Agent seconds include finalization overhead. Missing-row indices are zero-based. +The largest-reply count includes internal thinking tokens reported by the +provider, not only visible text. Scheme passed all 63 subcases in every run, +including the nested self-interpretation cases that failed in the baseline. +None of these Scheme runs exercised recovery. + +Known token-based estimated cost: **$0.906352500**. All six trials have complete +usage and cost accounting for reported tokens, with no incomplete model streams. +These estimates use the static DeepSeek price table and are not provider bills. + +## What happened after the observed truncation + +Model-extraction run 2 first inspected the environment. Its second response +consumed exactly 65,536 output tokens and ended with `max_tokens`, with no tool +calls to execute. This reproduces the relevant baseline pattern: long reasoning +before creating the required script. + +At **326.302 seconds**, with about **393.698 seconds** left in the original +720-second budget, the agent injected its single `truncation_recovery` prompt. +It then completed **15 more model responses** and **14 tool executions**, +including writing `/app/steal.py`, editing it, and running local checks. No +second truncation occurred. The native run completed at **552.428 seconds**; +neither the original deadline nor the reply/token caps were increased. + +The official verifier ran the submitted script and found its output file. +The script reported recovering **29 hidden neurons** from the verifier's +30-neuron network; original weight row **24** had no matching output row. +Therefore the official reward remained **0**. This was a solution-quality +failure after productive continuation, rather than the baseline's missing +script/output failure. The old stop-on-first-truncation rule would have stopped +at the same truncation boundary; this run supplies direct evidence that the +new branch continues beyond it. It does not supply a paired estimate of the +policy's effect on final task success. + +Model-extraction run 1 also created its script and output, then ended voluntarily +without truncation. Its official test failed on rows **2 and 27**. Run 3 passed +without truncating. The verifier replaces the visible 20-neuron example with a +30-neuron network; passing checks against the visible example alone does not +establish complete extraction for the official case. The observations support +investigating validation and extraction completeness separately from truncation +handling, but do not establish a single algorithmic root cause. + +## Interpretation and evidence + +The branch now has a real example of **one truncation followed by useful tool +work within the original budget**. This six-run sample has **zero passing +trials that used recovery**, so it does not show that recovery rescues the final +score. It also contains no Scheme truncation with which to test recovery on +that task. Keep these conclusions separate from the deterministic tests of the +once-per-run cap, partial-tool safety, and budget boundaries already recorded in +the initial experiment. + +The [structured results](../results/tb21-truncation-recovery-repeat3-20261005.json) +preserve every trial, its original result/trajectory/verifier-log paths, +per-attempt verifier records, source/artifact checks, accounting completeness, +and the sequence of model/tool activity after recovery. Raw artifacts remain +local in the referenced Git-ignored `jobs/` directories. + +The earlier 2/2 experiment remains a separate sample. The released v0.9.0 +[pilot20 baseline](v0.9.0-baseline-20261005.md) remains **18/20**. Fresh model +paths and different concurrent pairings prevent attributing aggregate score +differences to this change; three runs per task also provide only a small sample. +No extra runs were added after observing these results. diff --git a/benchmarks/harbor/results/tb21-truncation-recovery-repeat3-20261005.json b/benchmarks/harbor/results/tb21-truncation-recovery-repeat3-20261005.json new file mode 100644 index 0000000..6a88947 --- /dev/null +++ b/benchmarks/harbor/results/tb21-truncation-recovery-repeat3-20261005.json @@ -0,0 +1,880 @@ +{ + "schema_version": 1, + "generated_at": "2026-10-05T15:20:44.117868+00:00", + "status": "finished", + "agent_ref": "382cb38413d7d5368ad1be97e85142f280172a5b", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "experiment_plan": "benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/experiment.json", + "started_at": "2026-10-05T14:27:47.196300+00:00", + "finished_at": "2026-10-05T15:20:14.036548+00:00", + "summary": { + "planned": 6, + "finished": 6, + "passed": 4, + "scored_failures": 2, + "unscored": 0, + "truncated_responses": 1, + "recovery_events": 1, + "passed_with_recovery": 0, + "known_estimated_cost_usd": "0.906352500", + "cost_complete": true, + "usage_complete": true, + "incomplete_model_attempts": 0, + "by_task": [ + { + "task": "terminal-bench/model-extraction-relu-logits", + "planned": 3, + "finished": 3, + "passed": 1, + "scored_failures": 2, + "truncated_responses": 1, + "recovery_events": 1 + }, + { + "task": "terminal-bench/schemelike-metacircular-eval", + "planned": 3, + "finished": 3, + "passed": 3, + "scored_failures": 0, + "truncated_responses": 0, + "recovery_events": 0 + } + ] + }, + "jobs": [ + { + "job": "truncation-recovery-repeat3-scheme-1-20261005", + "config": "benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/scheme.json", + "attempt": 1, + "started_at": "2026-10-05T14:27:47.197456+00:00", + "exit_code": 0, + "finished_at": "2026-10-05T14:48:35.370423+00:00", + "wheel_sha256": "9f519124b62314d5b3625004441e0bfc29473533f29cef2dcf3640e55094800f", + "sources_match_plan": true + }, + { + "job": "truncation-recovery-repeat3-model-extraction-1-20261005", + "config": "benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/model-extraction.json", + "attempt": 1, + "started_at": "2026-10-05T14:27:47.198637+00:00", + "exit_code": 0, + "finished_at": "2026-10-05T14:37:00.961096+00:00", + "wheel_sha256": "9f519124b62314d5b3625004441e0bfc29473533f29cef2dcf3640e55094800f", + "sources_match_plan": true + }, + { + "job": "truncation-recovery-repeat3-model-extraction-2-20261005", + "config": "benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/model-extraction.json", + "attempt": 2, + "started_at": "2026-10-05T14:37:00.961578+00:00", + "exit_code": 0, + "finished_at": "2026-10-05T14:47:58.366693+00:00", + "wheel_sha256": "9f519124b62314d5b3625004441e0bfc29473533f29cef2dcf3640e55094800f", + "sources_match_plan": true + }, + { + "job": "truncation-recovery-repeat3-scheme-2-20261005", + "config": "benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/scheme.json", + "attempt": 2, + "started_at": "2026-10-05T14:47:58.367397+00:00", + "exit_code": 0, + "finished_at": "2026-10-05T15:14:52.234794+00:00", + "wheel_sha256": "9f519124b62314d5b3625004441e0bfc29473533f29cef2dcf3640e55094800f", + "sources_match_plan": true + }, + { + "job": "truncation-recovery-repeat3-model-extraction-3-20261005", + "config": "benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/model-extraction.json", + "attempt": 3, + "started_at": "2026-10-05T14:48:35.371179+00:00", + "exit_code": 0, + "finished_at": "2026-10-05T15:03:09.905596+00:00", + "wheel_sha256": "9f519124b62314d5b3625004441e0bfc29473533f29cef2dcf3640e55094800f", + "sources_match_plan": true + }, + { + "job": "truncation-recovery-repeat3-scheme-3-20261005", + "config": "benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/scheme.json", + "attempt": 3, + "started_at": "2026-10-05T15:03:09.906148+00:00", + "exit_code": 0, + "finished_at": "2026-10-05T15:20:14.035556+00:00", + "wheel_sha256": "9f519124b62314d5b3625004441e0bfc29473533f29cef2dcf3640e55094800f", + "sources_match_plan": true + } + ], + "preflight_trials": [ + { + "task": "terminal-bench/model-extraction-relu-logits", + "task_ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "task_checksum": "8b3048320aa04676089f9240563c2d7d381be03d8810bcf5f5d336cab538f523", + "agent_version": "0.9.1.dev5+g382cb3841", + "status": "setup_passed", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + } + }, + { + "task": "terminal-bench/schemelike-metacircular-eval", + "task_ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6", + "task_checksum": "e4525988ae20585b6b90a2f1af2fcc9d18d6363d3cfcdd7be5305199428883d2", + "agent_version": "0.9.1.dev5+g382cb3841", + "status": "setup_passed", + "verifier_preflight": { + "profile": null, + "status": "not_configured" + } + } + ], + "unchanged_task_files": 73, + "trials": [ + { + "attempt": 1, + "task": "terminal-bench/model-extraction-relu-logits", + "task_ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "task_checksum": "8b3048320aa04676089f9240563c2d7d381be03d8810bcf5f5d336cab538f523", + "trial": "model-extraction-relu-logits__UkLPYUv", + "job": "truncation-recovery-repeat3-model-extraction-1-20261005", + "agent_version": "0.9.1.dev5+g382cb3841", + "status": "scored", + "reward": 0.0, + "exception_type": null, + "result_path": "jobs/truncation-recovery-repeat3-model-extraction-1-20261005/model-extraction-relu-logits__UkLPYUv/result.json", + "agent_setup_seconds": 47.588, + "agent_execution_seconds": 414.411, + "verifier_seconds": 70.698, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "budget_seconds": 720, + "verifier_attempts": { + "timeout_sec": 900.0, + "max_attempts": 3, + "attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 70.34228242299287 + } + ] + }, + "completed_model_responses": 27, + "maximum_completion_tokens": 17791, + "trajectory_path": "jobs/truncation-recovery-repeat3-model-extraction-1-20261005/model-extraction-relu-logits__UkLPYUv/agent/trajectory.json", + "atif_valid": true, + "terminal": { + "status": "completed", + "duration_ms": 412719.058158, + "timestamp": "2026-10-05T14:35:33.693Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + }, + "metrics": { + "total_steps": 55, + "total_prompt_tokens": 1221130, + "total_completion_tokens": 62139, + "total_cached_tokens": 1205888, + "total_cost_usd": 0.086374728, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.086374728 + } + }, + "model_attempts": 27, + "truncated_responses": [], + "recovery_events": [], + "tools_executed_after_recovery": 0, + "incomplete_model_attempts": 0, + "known_cost_usd": 0.086374728, + "cost_complete": true, + "usage_complete": true, + "recovery_analysis": [], + "known_tokens": { + "prompt_tokens": 1221130, + "completion_tokens": 62139, + "cached_tokens": 1205888 + }, + "baseline": { + "reward": 0.0, + "terminal": { + "status": "completed", + "duration_ms": 299916.300286, + "timestamp": "2026-10-05T09:41:59.444Z", + "timestamp_source": "source_timestamp", + "outcome": "response_truncated" + }, + "agent_execution_seconds": 301.297, + "known_cost_usd": 0.079038732, + "cost_complete": true, + "usage_complete": true, + "trajectory_path": "jobs/v090-pilot20-720s-20261005/model-extraction-relu-logits__8mXWuta/agent/trajectory.json" + }, + "verification_observations": { + "log_path": "jobs/truncation-recovery-repeat3-model-extraction-1-20261005/model-extraction-relu-logits__UkLPYUv/verifier/test-stdout.txt", + "failure_kind": "unmatched_weight_rows", + "unmatched_rows": [ + 2, + 27 + ], + "script_and_output_file_present": true + } + }, + { + "attempt": 2, + "task": "terminal-bench/model-extraction-relu-logits", + "task_ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "task_checksum": "8b3048320aa04676089f9240563c2d7d381be03d8810bcf5f5d336cab538f523", + "trial": "model-extraction-relu-logits__v2wp9PK", + "job": "truncation-recovery-repeat3-model-extraction-2-20261005", + "agent_version": "0.9.1.dev5+g382cb3841", + "status": "scored", + "reward": 0.0, + "exception_type": null, + "result_path": "jobs/truncation-recovery-repeat3-model-extraction-2-20261005/model-extraction-relu-logits__v2wp9PK/result.json", + "agent_setup_seconds": 48.158, + "agent_execution_seconds": 554.062, + "verifier_seconds": 32.103, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "budget_seconds": 720, + "verifier_attempts": { + "timeout_sec": 900.0, + "max_attempts": 3, + "attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 31.75378994602943 + } + ] + }, + "completed_model_responses": 17, + "maximum_completion_tokens": 65536, + "trajectory_path": "jobs/truncation-recovery-repeat3-model-extraction-2-20261005/model-extraction-relu-logits__v2wp9PK/agent/trajectory.json", + "atif_valid": true, + "terminal": { + "status": "completed", + "duration_ms": 552428.102211, + "timestamp": "2026-10-05T14:47:09.879Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + }, + "metrics": { + "total_steps": 36, + "total_prompt_tokens": 460840, + "total_completion_tokens": 103311, + "total_cached_tokens": 454912, + "total_cost_usd": 0.128481072, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.128481072 + } + }, + "model_attempts": 17, + "truncated_responses": [ + { + "step_id": 5, + "timestamp": "2026-10-05T14:43:23.751Z", + "duration_ms": 323392.738675, + "completion_tokens": 65536 + } + ], + "recovery_events": [ + { + "step_id": 6, + "timestamp": "2026-10-05T14:43:23.754Z", + "model_call_id": "model-05968e30-2ac5-4ab7-9df8-e9e2e55115c2" + } + ], + "tools_executed_after_recovery": 14, + "incomplete_model_attempts": 0, + "known_cost_usd": 0.128481072, + "cost_complete": true, + "usage_complete": true, + "recovery_analysis": [ + { + "truncated_model_response_number": 2, + "truncated_step_id": 5, + "truncated_completion_tokens": 65536, + "unexecuted_truncated_tool_calls": 0, + "recovery_step_id": 6, + "elapsed_seconds": 326.302, + "remaining_work_seconds": 393.698, + "completed_responses_after_recovery": 15, + "subsequent_truncations": 0, + "subsequent_calls": [ + { + "step_id": 8, + "stop_reason": "tool_use", + "completion_tokens": 10684, + "incomplete": false, + "tools_requested": [ + "bash" + ], + "tool_observations": 1 + }, + { + "step_id": 10, + "stop_reason": "tool_use", + "completion_tokens": 8581, + "incomplete": false, + "tools_requested": [ + "write" + ], + "tool_observations": 1 + }, + { + "step_id": 12, + "stop_reason": "tool_use", + "completion_tokens": 59, + "incomplete": false, + "tools_requested": [ + "bash" + ], + "tool_observations": 1 + }, + { + "step_id": 14, + "stop_reason": "tool_use", + "completion_tokens": 234, + "incomplete": false, + "tools_requested": [ + "bash" + ], + "tool_observations": 1 + }, + { + "step_id": 16, + "stop_reason": "tool_use", + "completion_tokens": 5444, + "incomplete": false, + "tools_requested": [ + "write" + ], + "tool_observations": 1 + }, + { + "step_id": 18, + "stop_reason": "tool_use", + "completion_tokens": 203, + "incomplete": false, + "tools_requested": [ + "bash" + ], + "tool_observations": 1 + }, + { + "step_id": 20, + "stop_reason": "tool_use", + "completion_tokens": 1857, + "incomplete": false, + "tools_requested": [ + "bash" + ], + "tool_observations": 1 + }, + { + "step_id": 22, + "stop_reason": "tool_use", + "completion_tokens": 1038, + "incomplete": false, + "tools_requested": [ + "bash" + ], + "tool_observations": 1 + }, + { + "step_id": 24, + "stop_reason": "tool_use", + "completion_tokens": 1484, + "incomplete": false, + "tools_requested": [ + "bash" + ], + "tool_observations": 1 + }, + { + "step_id": 26, + "stop_reason": "tool_use", + "completion_tokens": 5945, + "incomplete": false, + "tools_requested": [ + "edit" + ], + "tool_observations": 1 + }, + { + "step_id": 28, + "stop_reason": "tool_use", + "completion_tokens": 643, + "incomplete": false, + "tools_requested": [ + "bash" + ], + "tool_observations": 1 + }, + { + "step_id": 30, + "stop_reason": "tool_use", + "completion_tokens": 208, + "incomplete": false, + "tools_requested": [ + "bash" + ], + "tool_observations": 1 + }, + { + "step_id": 32, + "stop_reason": "tool_use", + "completion_tokens": 590, + "incomplete": false, + "tools_requested": [ + "bash" + ], + "tool_observations": 1 + }, + { + "step_id": 34, + "stop_reason": "tool_use", + "completion_tokens": 261, + "incomplete": false, + "tools_requested": [ + "bash" + ], + "tool_observations": 1 + }, + { + "step_id": 36, + "stop_reason": "end_turn", + "completion_tokens": 416, + "incomplete": false, + "tools_requested": [], + "tool_observations": 0 + } + ] + } + ], + "known_tokens": { + "prompt_tokens": 460840, + "completion_tokens": 103311, + "cached_tokens": 454912 + }, + "baseline": { + "reward": 0.0, + "terminal": { + "status": "completed", + "duration_ms": 299916.300286, + "timestamp": "2026-10-05T09:41:59.444Z", + "timestamp_source": "source_timestamp", + "outcome": "response_truncated" + }, + "agent_execution_seconds": 301.297, + "known_cost_usd": 0.079038732, + "cost_complete": true, + "usage_complete": true, + "trajectory_path": "jobs/v090-pilot20-720s-20261005/model-extraction-relu-logits__8mXWuta/agent/trajectory.json" + }, + "verification_observations": { + "log_path": "jobs/truncation-recovery-repeat3-model-extraction-2-20261005/model-extraction-relu-logits__v2wp9PK/verifier/test-stdout.txt", + "failure_kind": "unmatched_weight_rows", + "unmatched_rows": [ + 24 + ], + "script_and_output_file_present": true + } + }, + { + "attempt": 3, + "task": "terminal-bench/model-extraction-relu-logits", + "task_ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "task_checksum": "8b3048320aa04676089f9240563c2d7d381be03d8810bcf5f5d336cab538f523", + "trial": "model-extraction-relu-logits__s8BNSNn", + "job": "truncation-recovery-repeat3-model-extraction-3-20261005", + "agent_version": "0.9.1.dev5+g382cb3841", + "status": "scored", + "reward": 1.0, + "exception_type": null, + "result_path": "jobs/truncation-recovery-repeat3-model-extraction-3-20261005/model-extraction-relu-logits__s8BNSNn/result.json", + "agent_setup_seconds": 128.006, + "agent_execution_seconds": 483.902, + "verifier_seconds": 237.859, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "budget_seconds": 720, + "verifier_attempts": { + "timeout_sec": 900.0, + "max_attempts": 3, + "attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 237.5025027259835 + } + ] + }, + "completed_model_responses": 23, + "maximum_completion_tokens": 53538, + "trajectory_path": "jobs/truncation-recovery-repeat3-model-extraction-3-20261005/model-extraction-relu-logits__s8BNSNn/agent/trajectory.json", + "atif_valid": true, + "terminal": { + "status": "completed", + "duration_ms": 482403.217558, + "timestamp": "2026-10-05T14:58:53.692Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + }, + "metrics": { + "total_steps": 47, + "total_prompt_tokens": 1607303, + "total_completion_tokens": 86571, + "total_cached_tokens": 1594624, + "total_cost_usd": 0.117256644, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.117256644 + } + }, + "model_attempts": 23, + "truncated_responses": [], + "recovery_events": [], + "tools_executed_after_recovery": 0, + "incomplete_model_attempts": 0, + "known_cost_usd": 0.117256644, + "cost_complete": true, + "usage_complete": true, + "recovery_analysis": [], + "known_tokens": { + "prompt_tokens": 1607303, + "completion_tokens": 86571, + "cached_tokens": 1594624 + }, + "baseline": { + "reward": 0.0, + "terminal": { + "status": "completed", + "duration_ms": 299916.300286, + "timestamp": "2026-10-05T09:41:59.444Z", + "timestamp_source": "source_timestamp", + "outcome": "response_truncated" + }, + "agent_execution_seconds": 301.297, + "known_cost_usd": 0.079038732, + "cost_complete": true, + "usage_complete": true, + "trajectory_path": "jobs/v090-pilot20-720s-20261005/model-extraction-relu-logits__8mXWuta/agent/trajectory.json" + }, + "verification_observations": { + "log_path": "jobs/truncation-recovery-repeat3-model-extraction-3-20261005/model-extraction-relu-logits__s8BNSNn/verifier/test-stdout.txt" + } + }, + { + "attempt": 1, + "task": "terminal-bench/schemelike-metacircular-eval", + "task_ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6", + "task_checksum": "e4525988ae20585b6b90a2f1af2fcc9d18d6363d3cfcdd7be5305199428883d2", + "trial": "schemelike-metacircular-eval__wUzkLEW", + "job": "truncation-recovery-repeat3-scheme-1-20261005", + "agent_version": "0.9.1.dev5+g382cb3841", + "status": "scored", + "reward": 1.0, + "exception_type": null, + "result_path": "jobs/truncation-recovery-repeat3-scheme-1-20261005/schemelike-metacircular-eval__wUzkLEW/result.json", + "agent_setup_seconds": 52.238, + "agent_execution_seconds": 1080.739, + "verifier_seconds": 93.716, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "budget_seconds": 2220, + "verifier_attempts": { + "timeout_sec": 2400.0, + "max_attempts": 3, + "attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 93.35175943694776 + } + ] + }, + "completed_model_responses": 41, + "maximum_completion_tokens": 17495, + "trajectory_path": "jobs/truncation-recovery-repeat3-scheme-1-20261005/schemelike-metacircular-eval__wUzkLEW/agent/trajectory.json", + "atif_valid": true, + "terminal": { + "status": "completed", + "duration_ms": 1079051.466026, + "timestamp": "2026-10-05T14:46:44.676Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + }, + "metrics": { + "total_steps": 83, + "total_prompt_tokens": 2732673, + "total_completion_tokens": 80099, + "total_cached_tokens": 2694656, + "total_cost_usd": 0.123691836, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.123691836 + } + }, + "model_attempts": 41, + "truncated_responses": [], + "recovery_events": [], + "tools_executed_after_recovery": 0, + "incomplete_model_attempts": 0, + "known_cost_usd": 0.123691836, + "cost_complete": true, + "usage_complete": true, + "recovery_analysis": [], + "known_tokens": { + "prompt_tokens": 2732673, + "completion_tokens": 80099, + "cached_tokens": 2694656 + }, + "baseline": { + "reward": 0.0, + "terminal": { + "status": "completed", + "duration_ms": 1161969.701524, + "timestamp": "2026-10-05T10:34:49.411Z", + "timestamp_source": "source_timestamp", + "outcome": "response_truncated" + }, + "agent_execution_seconds": 1163.351, + "known_cost_usd": 0.298166292, + "cost_complete": false, + "usage_complete": false, + "trajectory_path": "jobs/v090-pilot20-2220s-20261005/schemelike-metacircular-eval__RRuG7fV/agent/trajectory.json" + }, + "verification_observations": { + "log_path": "jobs/truncation-recovery-repeat3-scheme-1-20261005/schemelike-metacircular-eval__wUzkLEW/verifier/test-stdout.txt", + "subcases": { + "passed": 63, + "failed": 0, + "total": 63 + } + } + }, + { + "attempt": 2, + "task": "terminal-bench/schemelike-metacircular-eval", + "task_ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6", + "task_checksum": "e4525988ae20585b6b90a2f1af2fcc9d18d6363d3cfcdd7be5305199428883d2", + "trial": "schemelike-metacircular-eval__xDDbCB6", + "job": "truncation-recovery-repeat3-scheme-2-20261005", + "agent_version": "0.9.1.dev5+g382cb3841", + "status": "scored", + "reward": 1.0, + "exception_type": null, + "result_path": "jobs/truncation-recovery-repeat3-scheme-2-20261005/schemelike-metacircular-eval__xDDbCB6/result.json", + "agent_setup_seconds": 243.359, + "agent_execution_seconds": 1281.125, + "verifier_seconds": 65.292, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "budget_seconds": 2220, + "verifier_attempts": { + "timeout_sec": 2400.0, + "max_attempts": 3, + "attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 64.92442496900912 + } + ] + }, + "completed_model_responses": 79, + "maximum_completion_tokens": 35869, + "trajectory_path": "jobs/truncation-recovery-repeat3-scheme-2-20261005/schemelike-metacircular-eval__xDDbCB6/agent/trajectory.json", + "atif_valid": true, + "terminal": { + "status": "completed", + "duration_ms": 1279671.560734, + "timestamp": "2026-10-05T15:13:29.043Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + }, + "metrics": { + "total_steps": 159, + "total_prompt_tokens": 9423825, + "total_completion_tokens": 148648, + "total_cached_tokens": 9381760, + "total_cost_usd": 0.24728766, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.24728766 + } + }, + "model_attempts": 79, + "truncated_responses": [], + "recovery_events": [], + "tools_executed_after_recovery": 0, + "incomplete_model_attempts": 0, + "known_cost_usd": 0.24728766, + "cost_complete": true, + "usage_complete": true, + "recovery_analysis": [], + "known_tokens": { + "prompt_tokens": 9423825, + "completion_tokens": 148648, + "cached_tokens": 9381760 + }, + "baseline": { + "reward": 0.0, + "terminal": { + "status": "completed", + "duration_ms": 1161969.701524, + "timestamp": "2026-10-05T10:34:49.411Z", + "timestamp_source": "source_timestamp", + "outcome": "response_truncated" + }, + "agent_execution_seconds": 1163.351, + "known_cost_usd": 0.298166292, + "cost_complete": false, + "usage_complete": false, + "trajectory_path": "jobs/v090-pilot20-2220s-20261005/schemelike-metacircular-eval__RRuG7fV/agent/trajectory.json" + }, + "verification_observations": { + "log_path": "jobs/truncation-recovery-repeat3-scheme-2-20261005/schemelike-metacircular-eval__xDDbCB6/verifier/test-stdout.txt", + "subcases": { + "passed": 63, + "failed": 0, + "total": 63 + } + } + }, + { + "attempt": 3, + "task": "terminal-bench/schemelike-metacircular-eval", + "task_ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6", + "task_checksum": "e4525988ae20585b6b90a2f1af2fcc9d18d6363d3cfcdd7be5305199428883d2", + "trial": "schemelike-metacircular-eval__NW9oKWe", + "job": "truncation-recovery-repeat3-scheme-3-20261005", + "agent_version": "0.9.1.dev5+g382cb3841", + "status": "scored", + "reward": 1.0, + "exception_type": null, + "result_path": "jobs/truncation-recovery-repeat3-scheme-3-20261005/schemelike-metacircular-eval__NW9oKWe/result.json", + "agent_setup_seconds": 122.459, + "agent_execution_seconds": 829.948, + "verifier_seconds": 41.716, + "verifier_preflight": { + "profile": null, + "status": "not_configured" + }, + "budget_seconds": 2220, + "verifier_attempts": { + "timeout_sec": 2400.0, + "max_attempts": 3, + "attempts": [ + { + "attempt": 1, + "status": "completed", + "return_code": 0, + "duration_seconds": 41.33515577495564 + } + ] + }, + "completed_model_responses": 60, + "maximum_completion_tokens": 23002, + "trajectory_path": "jobs/truncation-recovery-repeat3-scheme-3-20261005/schemelike-metacircular-eval__NW9oKWe/agent/trajectory.json", + "atif_valid": true, + "terminal": { + "status": "completed", + "duration_ms": 828488.610808, + "timestamp": "2026-10-05T15:19:13.667Z", + "timestamp_source": "source_timestamp", + "outcome": "completed" + }, + "metrics": { + "total_steps": 121, + "total_prompt_tokens": 6525588, + "total_completion_tokens": 126214, + "total_cached_tokens": 6482560, + "total_cost_usd": 0.20326056, + "extra": { + "cost_is_estimated": true, + "estimated_cost_usd": 0.20326056 + } + }, + "model_attempts": 60, + "truncated_responses": [], + "recovery_events": [], + "tools_executed_after_recovery": 0, + "incomplete_model_attempts": 0, + "known_cost_usd": 0.20326056, + "cost_complete": true, + "usage_complete": true, + "recovery_analysis": [], + "known_tokens": { + "prompt_tokens": 6525588, + "completion_tokens": 126214, + "cached_tokens": 6482560 + }, + "baseline": { + "reward": 0.0, + "terminal": { + "status": "completed", + "duration_ms": 1161969.701524, + "timestamp": "2026-10-05T10:34:49.411Z", + "timestamp_source": "source_timestamp", + "outcome": "response_truncated" + }, + "agent_execution_seconds": 1163.351, + "known_cost_usd": 0.298166292, + "cost_complete": false, + "usage_complete": false, + "trajectory_path": "jobs/v090-pilot20-2220s-20261005/schemelike-metacircular-eval__RRuG7fV/agent/trajectory.json" + }, + "verification_observations": { + "log_path": "jobs/truncation-recovery-repeat3-scheme-3-20261005/schemelike-metacircular-eval__NW9oKWe/verifier/test-stdout.txt", + "subcases": { + "passed": 63, + "failed": 0, + "total": 63 + } + } + } + ], + "accounting_note": "Costs are token-based estimates using the static price table, not provider bills. Incomplete attempts retain unknown usage/cost, not zero.", + "comparison_note": "Three additional fresh trials per task, selected because both failed in the released baseline. Six standard jobs run from a predeclared queue with at most two simultaneous trials. Preserve all results separately from the earlier two fresh runs and frozen pilot20 baseline. No controlled replay or full-pilot score is implied. A pass without truncation does not exercise recovery; a pass after recovery is a live example, not an estimate of causal effect.", + "artifact_checks": { + "preflight_and_six_run_wheels_identical": true, + "wheel_sha256": "9f519124b62314d5b3625004441e0bfc29473533f29cef2dcf3640e55094800f", + "source_files_match_prior_experiment": 21, + "task_checksums_match_released_baseline": true, + "unchanged_task_files": 73, + "configuration_hashes_match_plan": true, + "task_images": [ + { + "image": "alexgshaw/model-extraction-relu-logits:20251031", + "id": "sha256:52fe1f089f38650f0dc22d7531ee6e01ebd526de1b349aaa2ecb331dca9fabce", + "matches_prior_experiment": true + }, + { + "image": "alexgshaw/schemelike-metacircular-eval:20251031", + "id": "sha256:b485be38bf80d56c1a4cd9da95bbf7ec129dc03f6786b86c0525518a96f6d1db", + "matches_prior_experiment": true + } + ] + }, + "interpretation": "One natural truncation, on the second model-extraction response, was followed by 15 completed model responses and 14 tool executions within the original budget. The official verifier still failed because one of 30 weight rows was missing. Live continuation is demonstrated; successful score recovery and a causal success-rate benefit are not demonstrated. Scheme passed all three runs without truncating." +} diff --git a/docs/dev_notes/en/0.10.x.md b/docs/dev_notes/en/0.10.x.md index ccad9ff..2ee152a 100644 --- a/docs/dev_notes/en/0.10.x.md +++ b/docs/dev_notes/en/0.10.x.md @@ -98,3 +98,62 @@ Keep the [experiment report](../../../benchmarks/harbor/reports/truncation-recov [structured results](../../../benchmarks/harbor/results/tb21-truncation-recovery-20261005.json), and [reproduction configuration](../../../benchmarks/harbor/configs/truncation-recovery-20261005/README.md). Do not keep rerunning until a favorable sample appears. + +### Follow-up: three runs per task to observe natural truncation recovery + +The maintainer subsequently explicitly requested three additional fresh runs +of each task. Commit the fixed six-job plan and queue order first, then execute +it on 2026-10-05 from 14:27:47 to 15:20:14 UTC with at most two concurrent tasks. +The run commit is `382cb38413d7d5368ad1be97e85142f280172a5b`; all 21 agent/adapter +Python files match the preceding experiment, with no runtime changes in this +round. Preserve the model, task revisions, images, 100 replies, 65,536 tokens, +720/2,220-second work budgets, and per-attempt verifier limits. + +Both installation preflights pass; both verifier dependency profiles are +`not_configured`. The preflight and all six live-run wheels are byte-identical, +all 73 task-file hashes remain unchanged, and task checksums match the released +baseline. All six native ATIF trajectories validate, and every official +verifier completes on its first attempt. There are no unscored trials. + +| Task | Run 1 | Run 2 | Run 3 | Truncations / recoveries | +| --- | --- | --- | --- | --- | +| Model extraction | Failed: weight rows 2 and 27 unmatched | Failed: weight row 24 unmatched | Passed | 1 / 1, in run 2 | +| Scheme | Passed, 63/63 | Passed, 63/63 | Passed, 63/63 | 0 / 0 | + +This round totals **4/6 passed, 2 scored failures, and 0 unscored trials**: +model extraction 1/3 and Scheme 3/3. Complete token-based estimated cost is +**$0.906352500**, not the actual bill; no model streams have missing usage. +Every native terminal outcome is `completed`, demonstrating that normal +termination does not itself mean the solution passes official verification. + +Model-extraction run 2 reproduces the key baseline situation: its second model +response consumes 65,536 tokens before creating the script. Inject one recovery +prompt at approximately 326.302 seconds, with about 393.698 seconds left in the +original 720-second budget. The agent then completes 15 more model responses +and 14 tool executions, writes and edits `/app/steal.py`, runs local checks, +and voluntarily completes at about 552.428 seconds. There is no second +truncation and no increase to the original budgets. + +Official verification runs the script and finds the output file, but it +recovers only 29 of 30 hidden neurons. Weight row 24 remains unmatched and the +reward is 0. **This live run demonstrates useful work after truncation, but does +not demonstrate recovery to a passing score.** The other failed extraction run +does not truncate and also creates both files, but misses two weight rows. The +official test replaces the visible 20-neuron example with a 30-neuron network, +so checking only the visible example cannot establish extraction completeness. +This suggests separately investigating validation coverage and algorithmic +completeness; this round does not establish the algorithmic root causes of all +failures. + +Preserve all six outcomes without rewriting the earlier 2/2 sample or v0.9.0's +18/20 baseline. None of the three Scheme runs truncates, so they still cannot +show whether recovery rescues Scheme. Nor can this small sample, with different +generation paths and concurrent pairings, establish a causal improvement in +success rate. Stop at the predeclared run count without adding trials after +observing the results. + +The [six-run report](../../../benchmarks/harbor/reports/truncation-recovery-repeat3-20261005.md), +[per-trial structured results](../../../benchmarks/harbor/results/tb21-truncation-recovery-repeat3-20261005.json), +and [fixed configuration](../../../benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/README.md) +preserve the recovery timestamp, remaining budget, subsequent model/tool +activity, and paths to the original logs for review. diff --git a/docs/dev_notes/zh-CN/0.10.x.md b/docs/dev_notes/zh-CN/0.10.x.md index 1051a26..a06aab7 100644 --- a/docs/dev_notes/zh-CN/0.10.x.md +++ b/docs/dev_notes/zh-CN/0.10.x.md @@ -76,3 +76,46 @@ usage/费用累计。锁定依赖时核心测试 351 通过、4 项可选传输 [结构化结果](../../../benchmarks/harbor/results/tb21-truncation-recovery-20261005.json) 与[复跑配置](../../../benchmarks/harbor/configs/truncation-recovery-20261005/README.md)， 不继续重跑直到获得有利样本。 + +### 追加实验：两题各运行三次，观察真实截断恢复 + +维护者随后明确要求两道题各追加三次全新运行。先提交固定六个 job 和队列顺序的 +实验计划，再于 2026-10-05 14:27:47–15:20:14 UTC 执行，最多同时运行两题。 +运行提交为 `382cb38413d7d5368ad1be97e85142f280172a5b`；21 个 agent/adapter +Python 文件与前轮实验完全一致，本轮没有修改运行逻辑。模型、任务版本、镜像、 +100 回合、65,536 token、720/2,220 秒工作预算和逐次 verifier 时限均保持不变。 + +两题安装预检通过，verifier 依赖 profile 均为 `not_configured`。预检及六次正式 +运行的 wheel 完全一致，73 个任务文件哈希保持不变，任务 checksum 与发布基线 +一致。六次原生 ATIF 全部有效，官方 verifier 均第一次完成，没有未评分试次。 + +| 题目 | 第 1 次 | 第 2 次 | 第 3 次 | 截断 / 恢复 | +| --- | --- | --- | --- | --- | +| 模型提取 | 失败：权重行 2、27 未匹配 | 失败：权重行 24 未匹配 | 通过 | 1 / 1，发生在第 2 次运行 | +| Scheme | 通过，63/63 | 通过，63/63 | 通过，63/63 | 0 / 0 | + +本轮合计 **4/6 通过、2 次评分失败、0 次未评分**；模型提取 1/3，Scheme 3/3。 +完整 token 费用估算为 **$0.906352500**，不是实际账单；没有 usage 缺失的模型流。 +所有原生终态均为 `completed`，说明正常结束本身并不等于解答通过官方验证。 + +模型提取第 2 次重现了基线的关键情形：第二次模型回复耗尽 65,536 token，尚未 +创建脚本。约 326.302 秒时注入一次恢复提示，原 720 秒预算还剩约 393.698 秒。 +随后又完成 15 次模型回复和 14 次工具执行，写出并修改 `/app/steal.py`、运行 +本地检查，最终于约 552.428 秒主动完成，没有再次截断，也没有增加原预算。 + +官方验证执行脚本并找到了输出文件，但只恢复出 30 个神经元中的 29 个，权重行 +24 未匹配，reward 仍为 0。**这次真实运行证明了截断后能够继续做有效工作，但 +尚未证明能把失败救成通过。** 另一次模型提取失败没有发生截断，脚本和输出也 +存在，只是有两行未匹配；官方测试会把可见的 20 神经元示例替换为 30 神经元网络， +因此仅验证可见示例不足以确认提取完整性。这提示后续可单独研究验证覆盖与算法 +完整性，但本轮没有确定全部失败的算法根因。 + +保留全部六次结果，先前的 2/2 样本及 0.9.0 的 18/20 baseline 均不改写。三次 +Scheme 都没有截断，仍不能据此判断恢复逻辑能否救回 Scheme；也不能从这批小 +样本、不同生成路径与并发配对推导策略对通过率的因果提升。按预定次数结束， +没有看到结果后再追加试次。 + +[六次运行报告](../../../benchmarks/harbor/reports/truncation-recovery-repeat3-20261005.md)、 +[逐次结构化结果](../../../benchmarks/harbor/results/tb21-truncation-recovery-repeat3-20261005.json) +和[固定配置](../../../benchmarks/harbor/configs/truncation-recovery-repeat3-20261005/README.md) +保留了恢复时间点、剩余预算、后续模型与工具活动，以及原始日志的复核路径。