From 3429556ab12a9a1b52b3cd4d6233a384420c2347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E9=9B=B2?= <137844255@qq.com> Date: Sun, 20 Sep 2026 09:33:45 +0800 Subject: [PATCH 1/2] chore(eval): put the evaluation harness, ledger and current results in the repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The evaluation rounds produced real findings (#67/#68/#69/#72/#81/#115 ...) but the harness and the numbers lived outside the repo: `jobs/` is gitignored and the reports were uncommitted, so a fresh checkout could neither reproduce a number nor see the trend. What lands here: - `scripts/eval/` — the L1 harness: 25 suites (`sweep.sh`), the mock provider with its fault-injection modes, the JSONL/ACP/trust/workflow/recovery probes, plus `summarize.py`, which turns a Harbor run into the committed per-task summary. New in this round: `workspace_lifetime_probe.py` (the #115 contract) and the `FILIFETIME=/FICMD64=` mock arguments it needs. - `docs/reports/evaluation-ledger.{md,json}` — one row per round with the protocol that makes the numbers comparable (source, harness, model, attempts/concurrency, task set) and the findings that left a permanent test behind. - `docs/reports/2026-09-19-v0.4.32-evaluation.md` + `…-tb2-v0432-summary.json` — the current round: L1 24/24 green, TB2 mean 0.7955 (70/88), and the two issues it filed (#114 argv self-kill, #115 workspace-lifetime service). Raw trajectories and container logs stay out of git (GBs, workspace content); the summary JSON carries one row per task (reward, exception, agent exit, turns, bytes). --- .../reports/2026-09-19-tb2-v0432-summary.json | 1181 +++++++++++++++++ docs/reports/2026-09-19-v0.4.32-evaluation.md | 128 ++ docs/reports/evaluation-ledger.json | 60 + docs/reports/evaluation-ledger.md | 54 + scripts/eval/acp_param_probe.py | 296 +++++ scripts/eval/command_contract.py | 400 ++++++ scripts/eval/context_probe.py | 164 +++ scripts/eval/daemon_probe.py | 367 +++++ scripts/eval/diff-runs.py | 110 ++ scripts/eval/exit_code_probe.py | 203 +++ scripts/eval/fault_injection.py | 333 +++++ scripts/eval/l3_batch.sh | 116 ++ scripts/eval/linux_shell_probe.py | 125 ++ scripts/eval/mcp_probe.py | 195 +++ scripts/eval/milestone_report.py | 212 +++ scripts/eval/mock_mcp_server.py | 124 ++ scripts/eval/mock_provider.py | 628 +++++++++ scripts/eval/permission_probe.py | 455 +++++++ scripts/eval/prompt_contract.py | 209 +++ scripts/eval/repo_task.py | 870 ++++++++++++ scripts/eval/run.sh | 79 ++ scripts/eval/sandbox_enforcement_probe.py | 149 +++ scripts/eval/sandbox_probe.py | 94 ++ scripts/eval/server_probe.py | 252 ++++ scripts/eval/server_restart_probe.py | 207 +++ scripts/eval/session_recovery_probe.py | 205 +++ scripts/eval/shell_edge_probe.py | 108 ++ scripts/eval/signal_probe.py | 101 ++ scripts/eval/storage_retention_probe.py | 208 +++ scripts/eval/subagent_probe.py | 244 ++++ scripts/eval/summarize.py | 250 ++++ scripts/eval/sweep.sh | 81 ++ scripts/eval/sweep_summary.py | 83 ++ scripts/eval/task_recovery_probe.py | 168 +++ scripts/eval/triage.py | 168 +++ scripts/eval/trust_probe.py | 114 ++ scripts/eval/trust_race_probe.py | 127 ++ scripts/eval/wire_param_probe.py | 267 ++++ scripts/eval/workflow_probe.py | 321 +++++ scripts/eval/workspace_lifetime_probe.py | 138 ++ 40 files changed, 9594 insertions(+) create mode 100644 docs/reports/2026-09-19-tb2-v0432-summary.json create mode 100644 docs/reports/2026-09-19-v0.4.32-evaluation.md create mode 100644 docs/reports/evaluation-ledger.json create mode 100644 docs/reports/evaluation-ledger.md create mode 100644 scripts/eval/acp_param_probe.py create mode 100644 scripts/eval/command_contract.py create mode 100644 scripts/eval/context_probe.py create mode 100644 scripts/eval/daemon_probe.py create mode 100644 scripts/eval/diff-runs.py create mode 100644 scripts/eval/exit_code_probe.py create mode 100644 scripts/eval/fault_injection.py create mode 100755 scripts/eval/l3_batch.sh create mode 100644 scripts/eval/linux_shell_probe.py create mode 100644 scripts/eval/mcp_probe.py create mode 100644 scripts/eval/milestone_report.py create mode 100644 scripts/eval/mock_mcp_server.py create mode 100644 scripts/eval/mock_provider.py create mode 100644 scripts/eval/permission_probe.py create mode 100644 scripts/eval/prompt_contract.py create mode 100644 scripts/eval/repo_task.py create mode 100755 scripts/eval/run.sh create mode 100644 scripts/eval/sandbox_enforcement_probe.py create mode 100644 scripts/eval/sandbox_probe.py create mode 100644 scripts/eval/server_probe.py create mode 100644 scripts/eval/server_restart_probe.py create mode 100644 scripts/eval/session_recovery_probe.py create mode 100644 scripts/eval/shell_edge_probe.py create mode 100644 scripts/eval/signal_probe.py create mode 100644 scripts/eval/storage_retention_probe.py create mode 100644 scripts/eval/subagent_probe.py create mode 100644 scripts/eval/summarize.py create mode 100755 scripts/eval/sweep.sh create mode 100644 scripts/eval/sweep_summary.py create mode 100644 scripts/eval/task_recovery_probe.py create mode 100644 scripts/eval/triage.py create mode 100644 scripts/eval/trust_probe.py create mode 100644 scripts/eval/trust_race_probe.py create mode 100644 scripts/eval/wire_param_probe.py create mode 100644 scripts/eval/workflow_probe.py create mode 100644 scripts/eval/workspace_lifetime_probe.py diff --git a/docs/reports/2026-09-19-tb2-v0432-summary.json b/docs/reports/2026-09-19-tb2-v0432-summary.json new file mode 100644 index 00000000..867f8561 --- /dev/null +++ b/docs/reports/2026-09-19-tb2-v0432-summary.json @@ -0,0 +1,1181 @@ +{ + "schema_version": 1, + "generated_at": "2026-09-20T01:32:05Z", + "run": "~/Documents/GitHub/orca-eval-reports/2026-09-19-tb2-v0432-artifacts/2026-09-19__23-31-23", + "protocol": { + "source_commit": "5970b40c", + "binary": "static x86_64-unknown-linux-musl release build", + "harness": "Harbor 0.20.0", + "agent": "terminal_bench.orca_agent:OrcaInstalledAgent", + "model": "deepseek-flash (routed: deepseek-v4-pro)", + "attempts_per_task": 1, + "concurrency": 6, + "excluded_tasks": [ + "terminal-bench/qemu-startup" + ], + "environment_build_timeout_multiplier": null, + "note": "mean_reward averages the trials that produced a verifier reward (Harbor's own denominator); pass_rate_of_all_trials uses every scheduled trial." + }, + "totals": { + "trials": 88, + "scored": 83, + "passed": 70, + "zeroed": 13, + "errored": 19, + "mean_reward": 0.7955, + "mean_reward_scored_only": 0.8434, + "pass_rate_of_all_trials": 0.7955, + "exceptions": { + "AgentTimeoutError": 13, + "EnvironmentStartTimeoutError": 4, + "NonZeroAgentExitCodeError": 1, + "RuntimeError": 1 + } + }, + "trials": [ + { + "task": "terminal-bench/adaptive-rejection-sampler", + "trial": "adaptive-rejection-sampler__cFvsL9B", + "reward": 0.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 31557263, + "turns": 62, + "agent_seconds": 687.0, + "verifier_seconds": 48.3 + }, + { + "task": "terminal-bench/bn-fit-modify", + "trial": "bn-fit-modify__mDEVm3x", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 4880187, + "turns": 29, + "agent_seconds": 213.2, + "verifier_seconds": 68.8 + }, + { + "task": "terminal-bench/break-filter-js-from-html", + "trial": "break-filter-js-from-html__aF7Qs7T", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 5053081, + "turns": 9, + "agent_seconds": 101.2, + "verifier_seconds": 67.6 + }, + { + "task": "terminal-bench/build-cython-ext", + "trial": "build-cython-ext__koBpYP4", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 6401995, + "turns": 74, + "agent_seconds": 445.5, + "verifier_seconds": 14.7 + }, + { + "task": "terminal-bench/build-pmars", + "trial": "build-pmars__m8bGdqt", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 8194794, + "turns": 72, + "agent_seconds": 332.6, + "verifier_seconds": 29.1 + }, + { + "task": "terminal-bench/build-pov-ray", + "trial": "build-pov-ray__y3CBUf2", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 5510989, + "turns": 59, + "agent_seconds": 426.8, + "verifier_seconds": 183.2 + }, + { + "task": "terminal-bench/caffe-cifar-10", + "trial": "caffe-cifar-10__WonbDXZ", + "reward": 0.0, + "exception": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 1200.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": 11060738, + "turns": null, + "agent_seconds": 1200.2, + "verifier_seconds": 595.3 + }, + { + "task": "terminal-bench/cancel-async-tasks", + "trial": "cancel-async-tasks__Laxt9hK", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 16987350, + "turns": 21, + "agent_seconds": 306.3, + "verifier_seconds": 95.9 + }, + { + "task": "terminal-bench/chess-best-move", + "trial": "chess-best-move__gnk9fAV", + "reward": 0.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 10336033, + "turns": 39, + "agent_seconds": 523.6, + "verifier_seconds": 53.0 + }, + { + "task": "terminal-bench/circuit-fibsqrt", + "trial": "circuit-fibsqrt__eWVqVkC", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 25034488, + "turns": 87, + "agent_seconds": 1490.2, + "verifier_seconds": 94.4 + }, + { + "task": "terminal-bench/cobol-modernization", + "trial": "cobol-modernization__ubXvAjp", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 21729083, + "turns": 48, + "agent_seconds": 637.3, + "verifier_seconds": 26.1 + }, + { + "task": "terminal-bench/code-from-image", + "trial": "code-from-image__PMqrNyu", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 21070579, + "turns": 60, + "agent_seconds": 665.3, + "verifier_seconds": 20.4 + }, + { + "task": "terminal-bench/compile-compcert", + "trial": "compile-compcert__kG6VXoi", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 18743890, + "turns": 136, + "agent_seconds": 1960.1, + "verifier_seconds": 56.2 + }, + { + "task": "terminal-bench/configure-git-webserver", + "trial": "configure-git-webserver__97JcKNk", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 5917236, + "turns": 16, + "agent_seconds": 132.8, + "verifier_seconds": 37.6 + }, + { + "task": "terminal-bench/constraints-scheduling", + "trial": "constraints-scheduling__L6gjPa9", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 2745733, + "turns": 6, + "agent_seconds": 49.9, + "verifier_seconds": 112.6 + }, + { + "task": "terminal-bench/count-dataset-tokens", + "trial": "count-dataset-tokens__uiUYrjC", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 6484224, + "turns": 28, + "agent_seconds": 379.0, + "verifier_seconds": 349.6 + }, + { + "task": "terminal-bench/crack-7z-hash", + "trial": "crack-7z-hash__H3LT8hK", + "reward": null, + "exception": "EnvironmentStartTimeoutError", + "exception_message": "Environment start timed out after 600.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": null, + "turns": null, + "agent_seconds": null, + "verifier_seconds": null + }, + { + "task": "terminal-bench/custom-memory-heap-crash", + "trial": "custom-memory-heap-crash__XdRnAhZ", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 11850209, + "turns": 28, + "agent_seconds": 290.2, + "verifier_seconds": 76.2 + }, + { + "task": "terminal-bench/db-wal-recovery", + "trial": "db-wal-recovery__oKB5id3", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 6906400, + "turns": 27, + "agent_seconds": 156.7, + "verifier_seconds": 81.5 + }, + { + "task": "terminal-bench/distribution-search", + "trial": "distribution-search__wFw9TPu", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 12178463, + "turns": 6, + "agent_seconds": 198.7, + "verifier_seconds": 103.8 + }, + { + "task": "terminal-bench/dna-assembly", + "trial": "dna-assembly__MZ8v9kc", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 32994017, + "turns": 43, + "agent_seconds": 860.0, + "verifier_seconds": 168.2 + }, + { + "task": "terminal-bench/dna-insert", + "trial": "dna-insert__n376Cfa", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 16855760, + "turns": 34, + "agent_seconds": 458.9, + "verifier_seconds": 61.4 + }, + { + "task": "terminal-bench/extract-elf", + "trial": "extract-elf__xR9ykMS", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 2397297, + "turns": 19, + "agent_seconds": 82.4, + "verifier_seconds": 134.1 + }, + { + "task": "terminal-bench/extract-moves-from-video", + "trial": "extract-moves-from-video__oPLC9xq", + "reward": 0.0, + "exception": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 1800.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": 19934506, + "turns": null, + "agent_seconds": 1800.4, + "verifier_seconds": 56.9 + }, + { + "task": "terminal-bench/feal-differential-cryptanalysis", + "trial": "feal-differential-cryptanalysis__6JfC6Dx", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 19231996, + "turns": 13, + "agent_seconds": 381.3, + "verifier_seconds": 50.8 + }, + { + "task": "terminal-bench/feal-linear-cryptanalysis", + "trial": "feal-linear-cryptanalysis__ocgcRvg", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 9323622, + "turns": 12, + "agent_seconds": 162.5, + "verifier_seconds": 71.5 + }, + { + "task": "terminal-bench/filter-js-from-html", + "trial": "filter-js-from-html__kpij9RZ", + "reward": 1.0, + "exception": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 1800.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": 18777988, + "turns": null, + "agent_seconds": 1800.5, + "verifier_seconds": 829.9 + }, + { + "task": "terminal-bench/financial-document-processor", + "trial": "financial-document-processor__WfSziNn", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 3322330, + "turns": 25, + "agent_seconds": 226.2, + "verifier_seconds": 43.1 + }, + { + "task": "terminal-bench/fix-code-vulnerability", + "trial": "fix-code-vulnerability__kuVS9G9", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 700710, + "turns": 9, + "agent_seconds": 21.2, + "verifier_seconds": 9.0 + }, + { + "task": "terminal-bench/fix-git", + "trial": "fix-git__LZhFN4n", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 1072958, + "turns": 8, + "agent_seconds": 24.8, + "verifier_seconds": 37.9 + }, + { + "task": "terminal-bench/fix-ocaml-gc", + "trial": "fix-ocaml-gc__6VL3z32", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 12752100, + "turns": 85, + "agent_seconds": 1579.4, + "verifier_seconds": 1093.0 + }, + { + "task": "terminal-bench/gcode-to-text", + "trial": "gcode-to-text__iKV77Mv", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 23609638, + "turns": 81, + "agent_seconds": 709.9, + "verifier_seconds": 28.5 + }, + { + "task": "terminal-bench/git-leak-recovery", + "trial": "git-leak-recovery__Eiver2j", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 1238387, + "turns": 9, + "agent_seconds": 28.3, + "verifier_seconds": 118.7 + }, + { + "task": "terminal-bench/git-multibranch", + "trial": "git-multibranch__HuKGiEc", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 3038661, + "turns": 18, + "agent_seconds": 88.9, + "verifier_seconds": 157.8 + }, + { + "task": "terminal-bench/gpt2-codegolf", + "trial": "gpt2-codegolf__LS86qVP", + "reward": 1.0, + "exception": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 900.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": 47007718, + "turns": null, + "agent_seconds": 901.0, + "verifier_seconds": 43.8 + }, + { + "task": "terminal-bench/headless-terminal", + "trial": "headless-terminal__7aL5hYA", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 11049880, + "turns": 39, + "agent_seconds": 357.6, + "verifier_seconds": 19.8 + }, + { + "task": "terminal-bench/hf-model-inference", + "trial": "hf-model-inference__7NzpwSS", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 1071893, + "turns": 15, + "agent_seconds": 126.9, + "verifier_seconds": 25.0 + }, + { + "task": "terminal-bench/install-windows-3-11", + "trial": "install-windows-3-11__vSVBmwG", + "reward": 0.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 25016019, + "turns": 65, + "agent_seconds": 810.0, + "verifier_seconds": 226.6 + }, + { + "task": "terminal-bench/kv-store-grpc", + "trial": "kv-store-grpc__8rmWwgF", + "reward": 0.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 856825, + "turns": 11, + "agent_seconds": 42.8, + "verifier_seconds": 31.6 + }, + { + "task": "terminal-bench/large-scale-text-editing", + "trial": "large-scale-text-editing__67DyrDZ", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 12493219, + "turns": 31, + "agent_seconds": 481.0, + "verifier_seconds": 66.0 + }, + { + "task": "terminal-bench/largest-eigenval", + "trial": "largest-eigenval__mhYvsJn", + "reward": 1.0, + "exception": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 900.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": 30987053, + "turns": null, + "agent_seconds": 900.6, + "verifier_seconds": 17.2 + }, + { + "task": "terminal-bench/llm-inference-batching-scheduler", + "trial": "llm-inference-batching-scheduler__Mgga25H", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 26511082, + "turns": 30, + "agent_seconds": 552.4, + "verifier_seconds": 21.2 + }, + { + "task": "terminal-bench/log-summary-date-ranges", + "trial": "log-summary-date-ranges__EJmZDLj", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 743708, + "turns": 8, + "agent_seconds": 27.1, + "verifier_seconds": 31.2 + }, + { + "task": "terminal-bench/mailman", + "trial": "mailman__au8tULW", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 11471267, + "turns": 44, + "agent_seconds": 398.1, + "verifier_seconds": 63.7 + }, + { + "task": "terminal-bench/make-doom-for-mips", + "trial": "make-doom-for-mips__MxfLfGC", + "reward": 0.0, + "exception": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 900.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": 31239198, + "turns": null, + "agent_seconds": 900.6, + "verifier_seconds": 85.6 + }, + { + "task": "terminal-bench/make-mips-interpreter", + "trial": "make-mips-interpreter__NHBzWWR", + "reward": 0.0, + "exception": "NonZeroAgentExitCodeError", + "exception_message": "Command failed (exit 137): orca exec --mode full-auto --output-format jsonl -- 'I have provided /app/doomgeneric_mips, a MIPS elf file, along with doomgeneric/, the corresponding source code. Please implement a MIPS interpreter complete with handling system calls called vm.js so that I can run `node", + "agent_exit_code": 137, + "agent_terminal": null, + "trajectory_bytes": 26109969, + "turns": null, + "agent_seconds": 635.1, + "verifier_seconds": 126.3 + }, + { + "task": "terminal-bench/mcmc-sampling-stan", + "trial": "mcmc-sampling-stan__AiuFDvh", + "reward": 1.0, + "exception": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 1800.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": 3601932, + "turns": null, + "agent_seconds": 1800.2, + "verifier_seconds": 306.4 + }, + { + "task": "terminal-bench/merge-diff-arc-agi-task", + "trial": "merge-diff-arc-agi-task__NxYB8Wr", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 4882835, + "turns": 20, + "agent_seconds": 111.3, + "verifier_seconds": 104.7 + }, + { + "task": "terminal-bench/model-extraction-relu-logits", + "trial": "model-extraction-relu-logits__67GRuP5", + "reward": 1.0, + "exception": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 900.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": 44749831, + "turns": null, + "agent_seconds": 901.0, + "verifier_seconds": 84.0 + }, + { + "task": "terminal-bench/modernize-scientific-stack", + "trial": "modernize-scientific-stack__595DmhJ", + "reward": 0.0, + "exception": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 600.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": 0, + "turns": null, + "agent_seconds": 600.1, + "verifier_seconds": 33.8 + }, + { + "task": "terminal-bench/mteb-leaderboard", + "trial": "mteb-leaderboard__dTCyYRg", + "reward": null, + "exception": "EnvironmentStartTimeoutError", + "exception_message": "Environment start timed out after 600.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": null, + "turns": null, + "agent_seconds": null, + "verifier_seconds": null + }, + { + "task": "terminal-bench/mteb-retrieve", + "trial": "mteb-retrieve__H3WrZ49", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 2230817, + "turns": 16, + "agent_seconds": 250.1, + "verifier_seconds": 298.0 + }, + { + "task": "terminal-bench/multi-source-data-merger", + "trial": "multi-source-data-merger__G8YqPtj", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 1129107, + "turns": 10, + "agent_seconds": 42.9, + "verifier_seconds": 66.9 + }, + { + "task": "terminal-bench/nginx-request-logging", + "trial": "nginx-request-logging__ACRK4rn", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 923538, + "turns": 13, + "agent_seconds": 41.9, + "verifier_seconds": 34.7 + }, + { + "task": "terminal-bench/openssl-selfsigned-cert", + "trial": "openssl-selfsigned-cert__jNm9z7D", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 2080475, + "turns": 9, + "agent_seconds": 42.5, + "verifier_seconds": 71.4 + }, + { + "task": "terminal-bench/overfull-hbox", + "trial": "overfull-hbox__BSQBRzJ", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 10909818, + "turns": 40, + "agent_seconds": 258.7, + "verifier_seconds": 160.1 + }, + { + "task": "terminal-bench/password-recovery", + "trial": "password-recovery__T28WeuV", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 3960616, + "turns": 16, + "agent_seconds": 76.3, + "verifier_seconds": 35.0 + }, + { + "task": "terminal-bench/path-tracing", + "trial": "path-tracing__YRMhFpG", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 8529994, + "turns": 33, + "agent_seconds": 171.5, + "verifier_seconds": 87.3 + }, + { + "task": "terminal-bench/path-tracing-reverse", + "trial": "path-tracing-reverse__XdDixBz", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 7226072, + "turns": 36, + "agent_seconds": 170.1, + "verifier_seconds": 41.6 + }, + { + "task": "terminal-bench/polyglot-c-py", + "trial": "polyglot-c-py__u8ovpLQ", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 10373788, + "turns": 20, + "agent_seconds": 325.6, + "verifier_seconds": 93.0 + }, + { + "task": "terminal-bench/polyglot-rust-c", + "trial": "polyglot-rust-c__iwsvypj", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 10321959, + "turns": 12, + "agent_seconds": 184.6, + "verifier_seconds": 117.6 + }, + { + "task": "terminal-bench/portfolio-optimization", + "trial": "portfolio-optimization__UZ7RSFh", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 5279305, + "turns": 20, + "agent_seconds": 252.4, + "verifier_seconds": 97.5 + }, + { + "task": "terminal-bench/protein-assembly", + "trial": "protein-assembly__YEHYPWr", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 22135577, + "turns": 40, + "agent_seconds": 553.7, + "verifier_seconds": 49.8 + }, + { + "task": "terminal-bench/prove-plus-comm", + "trial": "prove-plus-comm__2UcitjT", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 400020, + "turns": 7, + "agent_seconds": 12.2, + "verifier_seconds": 64.5 + }, + { + "task": "terminal-bench/pypi-server", + "trial": "pypi-server__x4gfHwY", + "reward": 0.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 2317312, + "turns": 19, + "agent_seconds": 87.5, + "verifier_seconds": 31.4 + }, + { + "task": "terminal-bench/pytorch-model-cli", + "trial": "pytorch-model-cli__yfWP5Ny", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 8091846, + "turns": 42, + "agent_seconds": 273.5, + "verifier_seconds": 166.9 + }, + { + "task": "terminal-bench/pytorch-model-recovery", + "trial": "pytorch-model-recovery__fgx5Nyg", + "reward": null, + "exception": "EnvironmentStartTimeoutError", + "exception_message": "Environment start timed out after 600.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": null, + "turns": null, + "agent_seconds": null, + "verifier_seconds": null + }, + { + "task": "terminal-bench/qemu-alpine-ssh", + "trial": "qemu-alpine-ssh__dHPaTjt", + "reward": 0.0, + "exception": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 900.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": 15947369, + "turns": null, + "agent_seconds": 900.4, + "verifier_seconds": 8.3 + }, + { + "task": "terminal-bench/query-optimize", + "trial": "query-optimize__U9Y7r2n", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 12618062, + "turns": 39, + "agent_seconds": 324.7, + "verifier_seconds": 564.7 + }, + { + "task": "terminal-bench/raman-fitting", + "trial": "raman-fitting__6Y2oy4L", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 14732833, + "turns": 23, + "agent_seconds": 340.4, + "verifier_seconds": 25.9 + }, + { + "task": "terminal-bench/regex-chess", + "trial": "regex-chess__XvB5vs5", + "reward": 1.0, + "exception": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 3600.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": 58596482, + "turns": null, + "agent_seconds": 3601.2, + "verifier_seconds": 114.2 + }, + { + "task": "terminal-bench/regex-log", + "trial": "regex-log__A4m6sPX", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 7391811, + "turns": 12, + "agent_seconds": 124.5, + "verifier_seconds": 119.7 + }, + { + "task": "terminal-bench/reshard-c4-data", + "trial": "reshard-c4-data__wzgpTFJ", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 7101471, + "turns": 23, + "agent_seconds": 171.6, + "verifier_seconds": 103.3 + }, + { + "task": "terminal-bench/rstan-to-pystan", + "trial": "rstan-to-pystan__Xdm9CQW", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 15798472, + "turns": 87, + "agent_seconds": 1562.8, + "verifier_seconds": 51.4 + }, + { + "task": "terminal-bench/sam-cell-seg", + "trial": "sam-cell-seg__QZsiLXU", + "reward": 0.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 24936964, + "turns": 125, + "agent_seconds": 2552.8, + "verifier_seconds": 89.9 + }, + { + "task": "terminal-bench/sanitize-git-repo", + "trial": "sanitize-git-repo__u39RmQY", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 4253625, + "turns": 20, + "agent_seconds": 97.5, + "verifier_seconds": 28.0 + }, + { + "task": "terminal-bench/schemelike-metacircular-eval", + "trial": "schemelike-metacircular-eval__hVDqYw6", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 51260131, + "turns": 106, + "agent_seconds": 1562.2, + "verifier_seconds": 75.0 + }, + { + "task": "terminal-bench/sparql-university", + "trial": "sparql-university__bHDvSr6", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 4140900, + "turns": 18, + "agent_seconds": 150.2, + "verifier_seconds": 44.0 + }, + { + "task": "terminal-bench/sqlite-db-truncate", + "trial": "sqlite-db-truncate__wRPP3Hc", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 4701656, + "turns": 6, + "agent_seconds": 72.4, + "verifier_seconds": 24.2 + }, + { + "task": "terminal-bench/sqlite-with-gcov", + "trial": "sqlite-with-gcov__4qnqdSj", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 1798468, + "turns": 22, + "agent_seconds": 157.4, + "verifier_seconds": 71.6 + }, + { + "task": "terminal-bench/torch-pipeline-parallelism", + "trial": "torch-pipeline-parallelism__9S6XrbM", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 17090769, + "turns": 45, + "agent_seconds": 612.3, + "verifier_seconds": 628.6 + }, + { + "task": "terminal-bench/torch-tensor-parallelism", + "trial": "torch-tensor-parallelism__HJbU9uL", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 14697284, + "turns": 38, + "agent_seconds": 552.7, + "verifier_seconds": 636.5 + }, + { + "task": "terminal-bench/train-fasttext", + "trial": "train-fasttext__yDkiLTa", + "reward": 0.0, + "exception": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 3600.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": 30849365, + "turns": null, + "agent_seconds": 3600.7, + "verifier_seconds": 62.5 + }, + { + "task": "terminal-bench/tune-mjcf", + "trial": "tune-mjcf__Wc593n8", + "reward": 1.0, + "exception": "AgentTimeoutError", + "exception_message": "Agent execution timed out after 900.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": 9125006, + "turns": null, + "agent_seconds": 900.2, + "verifier_seconds": 52.3 + }, + { + "task": "terminal-bench/video-processing", + "trial": "video-processing__cNGFYxf", + "reward": null, + "exception": "EnvironmentStartTimeoutError", + "exception_message": "Environment start timed out after 600.0 seconds", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": null, + "turns": null, + "agent_seconds": null, + "verifier_seconds": null + }, + { + "task": "terminal-bench/vulnerable-secret", + "trial": "vulnerable-secret__KPsnEuW", + "reward": null, + "exception": "RuntimeError", + "exception_message": "Docker compose command failed for environment vulnerable-secret. Command: docker compose --project-name vulnerable-secret__kpsneuw__env --project-directory ~/.cache/harbor/tasks/packages/terminal-bench/vulnerable-secret/2a8a7b743f4f42fe70c6aadfde5621fceaf8ceb343331586f4479485bc1f8da0/en", + "agent_exit_code": null, + "agent_terminal": null, + "trajectory_bytes": null, + "turns": null, + "agent_seconds": null, + "verifier_seconds": null + }, + { + "task": "terminal-bench/winning-avg-corewars", + "trial": "winning-avg-corewars__YGqLGqH", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 35155904, + "turns": 115, + "agent_seconds": 1571.9, + "verifier_seconds": 47.7 + }, + { + "task": "terminal-bench/write-compressor", + "trial": "write-compressor__hUB7sGb", + "reward": 1.0, + "exception": null, + "exception_message": null, + "agent_exit_code": 0, + "agent_terminal": "success", + "trajectory_bytes": 22830780, + "turns": 21, + "agent_seconds": 424.8, + "verifier_seconds": 396.4 + } + ] +} diff --git a/docs/reports/2026-09-19-v0.4.32-evaluation.md b/docs/reports/2026-09-19-v0.4.32-evaluation.md new file mode 100644 index 00000000..abe9287e --- /dev/null +++ b/docs/reports/2026-09-19-v0.4.32-evaluation.md @@ -0,0 +1,128 @@ +# Orca 评测报告 — v0.4.32（2026-09-19） + +来源 `origin/main` @ `5970b40c`（v0.4.32，含本轮 26 个 issue 修复与你那批 CI 稳定化），从源码重建 host +与 musl 二进制，跑 L1 全量套件 + Terminal-Bench 2.0 全量（88 题）。 + +## 协议（数字可比的前提） + +| 项 | 值 | +|---|---| +| Source | `5970b40c`（tag 附近，v0.4.32） | +| Binary | 静态 `x86_64-unknown-linux-musl`（`messense/rust-musl-cross` 构建）+ host `target/release/orca` | +| Harness | Harbor 0.20.0，`terminal_bench.orca_agent:OrcaInstalledAgent`（仓库内 adapter） | +| Model | adapter 默认 → routed `deepseek-v4-pro`（`reason: default_pro`） | +| Settings | `-k 1 -n 6`，一题一次尝试 | +| Tasks | 88 / 89，`terminal-bench/qemu-startup` 按 `terminal_bench/quarantine.json` 排除 | +| Window | 2026-09-19 23:31 → 2026-09-20 03:13（本地，含 13 个 900–3600 s 长任务） | + +## 结论速览 + +| 层 | 结果 | 对比 | +|---|---|---| +| L1（24 套件） | **全绿**，0 失败、0 "blocked by" 注解 | 上一基线 155 通过 / 32 失败 | +| TB2 2.0 | **mean 0.7955**（70/88）；计分题内均值 0.8434 | v0.4.31 修正后 0.761 | +| 新 issue | **#114**、**#115** | 之前 0 个 open | + +明细：[`2026-09-19-tb2-v0432-summary.json`](2026-09-19-tb2-v0432-summary.json)（每题 reward / exception / +agent exit / turns / 轨迹字节 / 耗时），台账：[`evaluation-ledger.md`](evaluation-ledger.md)。 + +## L1：24/24 套件全绿 + +`bash scripts/eval/sweep.sh jobs/eval-sweep/20260919-v0432`（约 40 分钟，无网络/无 API key）。 +上一轮为红、这轮转绿的关键项： + +- `fault_injection.py` **13/13**（#67 重复 tool-call id、#68 5xx 重试、#62 截断重试） +- `wire_param_probe.py` **14/14**（#77 裸字符串 input、#78 请求 id 保留、#79 请求错误不再杀 server） +- `acp_param_probe.py` **20/20**（#80 参数错误 `-32602`）、`daemon_probe.py` 12/12 +- `signal_probe.py` **2/2**（#72：130/143 + terminal + 无孤儿） +- `permission_probe.py` 16/16（#74）、`trust_probe.py` 8/8 + `trust_race_probe.py`（#75/#81） +- `workflow_probe.py` 19/19（#82–#85）、`linux_shell_probe.py`（#70）、`sandbox_probe.py`（#69） + +## TB2 2.0：mean 0.7955 + +| 分类 | 数量 | 说明 | +|---|---|---| +| reward 1.0 | 70 | | +| reward 0.0 | 13 | 其中 3 题是本报告的两个产品缺陷所致，6 题是超时，4 题是纯能力 | +| `AgentTimeoutError` | 13 | 其中 **7 题验证器已通过**（活儿干完了没收手）：`filter-js-from-html`、`gpt2-codegolf`、`largest-eigenval`、`mcmc-sampling-stan`、`model-extraction-relu-logits`、`regex-chess`、`tune-mjcf` | +| `EnvironmentStartTimeoutError` | 4 | 本机 Docker 6 并发噪音，已复跑（见下） | +| `NonZeroAgentExitCodeError` | 1 | exit 137 = **#114 自我 kill** | +| `RuntimeError` | 1 | 拉镜像 EOF，环境噪音 | + +### 产品缺陷 1：instruction 在 argv 里，进程匹配命令杀掉 agent 自己（#114） + +`terminal-bench/make-mips-interpreter`：agent 用 `/proc` 扫描并 `kill -9` cmdline 含 `vm.js` 的进程， +而 `orca exec … -- ''` 的 argv 里正带着含 `vm.js` 的指令 → 自杀，exit 137，26 MB 轨迹断在 +第 43 轮，TB2 记成 agent error。issue 内给出 19 行本地复现（`pgrep -fl $MARKER` 能看到 orca 自己， +`pkill -9 -f $MARKER` → 137）。建议 adapter 把 prompt 走 stdin（`printf '%s' "$instruction" | orca exec …`）。 + +### 产品缺陷 2：`lifetime: "workspace"` 的服务在会话结束时被杀（#115） + +文档承诺 *"It survives the task that started it and is stopped explicitly"*，任务取消路径也照此过滤 +（`crates/orca-runtime/src/tasks.rs:5185`），但会话结束走的 `RuntimeShellSessionManager::terminate_all()` +（`crates/orca-runtime/src/shell_session.rs:955`，由 `terminal_service.rs:665/672` 触发）对所有 shell 一律 +`Cancelled`，不看 lifetime。受害任务：`install-windows-3-11`（QEMU+VNC，会话内已用 QMP screendump 验证画面）、 +`kv-store-grpc`（`server.py` 会话内客户端往返成功）。新探针 +[`scripts/eval/workspace_lifetime_probe.py`](../../scripts/eval/workspace_lifetime_probe.py) 直接测契约： + +``` +[FAIL] lifetime=workspace exit=0 terminal=True ticks_at_exit=31 ticks_after=31 — 会话结束后应继续运行 +[PASS] lifetime=task exit=0 terminal=True ticks_at_exit=31 ticks_after=31 — 随会话停止（符合预期） +``` + +### 基础设施噪音与复跑 + +4 次 `EnvironmentStartTimeoutError`（`pytorch-model-recovery`、`mteb-leaderboard`、`crack-7z-hash`、 +`video-processing`）+ 1 次拉镜像 EOF（`vulnerable-secret`）在 `-n 1` 下逐题复跑：**5/5 验证器通过** +（其中 `pytorch-model-recovery` 满分，等于 #61 的 `--` prompt 修复在 TB2 端到端确认）。按验证器通过计入， +本轮解法率为 **75/88 = 85.2%**（harbor 正式 mean 仍按全部 88 题计为 0.7955）。 + +## Agent 水平定位 + +| 维度 | 数值 / 题目 | +|---|---| +| TB2 mean（单次尝试） | 0.7955（70/88）；计分题内 0.8434 | +| 三连跑（同协议） | 0.494 → 0.761 → **0.7955** | +| 超时 | 13（15%），其中 7 题验证器已通过 | +| 产品缺陷丢分 | 3 题（#114 ×1、#115 ×2） | +| 纯能力不足 | 4 题（`chess-best-move`、`pypi-server`、`sam-cell-seg`、`adaptive-rejection-sampler`） | +| 可靠性 | L1 24/24；TB2 无 orca 崩溃/泄漏/轨迹丢失 | + +- **工程可靠性是生产级**：协议、ACP、信号、信任并发、工作流、沙箱、MCP、恢复、保留全部有自动化契约； + TB2 里没有一次崩溃类错误。 +- **能力层约 4/5 任务可解**：70 题覆盖仓库恢复/编辑、构建、协议与服务实现、密码学/逆向、数据处理。 +- **瓶颈是长时程效率**：13 次超时里 7 次任务已达标只是没停手，6 次是真算不完（训练、模拟器、大编译）。 + +外部对比：TB2 有公开榜（[tbench.ai](https://www.tbench.ai/leaderboard/terminal-bench/2.0)、 +[benchlm.ai](https://benchlm.ai/benchmarks/terminal-bench-2)、[BenchmarkList](https://benchmarklist.com/benchmarks/vals_terminal_bench_2/)）， +但本机网络把外网域名解析到非公网 IP、抓不到具体数值，因此本报告不给不可核实的对比；且公开条目多为 +k≥5 + 厂商 scaffold，与本仓库"单次尝试 + `orca exec` + routed `deepseek-v4-pro`"不同口径。 + +现实上限估计 **0.87–0.90**：修 #114/#115 立即回收 3 题（≈ +3.4 pt），超时池 13 题靠收尾纪律与 effort +实验逐步回收，剩下 4 题属模型能力。 + +## 复现 + +```bash +# L1（无网络） +bash scripts/eval/sweep.sh jobs/eval-sweep/$(date +%Y%m%d) + +# TB2（Docker + API key） +harbor run -d "terminal-bench/terminal-bench-2" \ + --agent "terminal_bench.orca_agent:OrcaInstalledAgent" \ + -k 1 -n 6 --exclude-task-name "terminal-bench/qemu-startup" \ + --mounts '[{"type":"bind","source":"'"$PWD"'/target/x86_64-unknown-linux-musl/release","target":"/mnt/orca-bin","read_only":true}]' + +# 产出可提交的 summary（原始 jobs/ 与轨迹不入库） +python3 scripts/eval/summarize.py jobs/full-88-v0432/ \ + --out docs/reports/2026-09-19-tb2-v0432-summary.json --source-commit 5970b40c \ + --model "deepseek-flash (routed: deepseek-v4-pro)" -k 1 -n 6 +``` + +## 产物说明 + +- **入库**：本报告 + [`summary.json`](2026-09-19-tb2-v0432-summary.json)（34 KB，88 行 per-task）+ + [`evaluation-ledger.md`](evaluation-ledger.md) / [`.json`](evaluation-ledger.json) + `scripts/eval/` 夹具。 +- **不入库**：`jobs/` 原始轨迹与容器日志（GB 级、含 workspace 内容）。三个关键题目的完整轨迹 + （#114 的 `make-mips-interpreter`、#115 的 `install-windows-3-11` 与 `kv-store-grpc`）已 gzip 归档在 + 仓库外，需要时可附到对应 issue。 diff --git a/docs/reports/evaluation-ledger.json b/docs/reports/evaluation-ledger.json new file mode 100644 index 00000000..0a141530 --- /dev/null +++ b/docs/reports/evaluation-ledger.json @@ -0,0 +1,60 @@ +{ + "schema_version": 1, + "updated_at": "2026-09-19", + "definition": { + "tb2_mean": "Harbor aggregation over every scheduled trial; a trial without a verifier reward counts as 0", + "tb2_mean_scored_only": "mean over the trials that produced a verifier reward", + "l1": "scripts/eval/sweep.sh suites, no network or API key required" + }, + "entries": [ + { + "date": "2026-09-15", + "source_commit": "3aa4efdfa", + "version": "v0.4.31", + "tasks": 89, + "settings": { "attempts": 1, "concurrency": 3, "environment_build_timeout_multiplier": 2.0 }, + "tb2_mean": 0.494, + "tb2_passed": 44, + "report": "docs/reports/2026-09-15-terminal-bench-2-v0.4.31.md" + }, + { + "date": "2026-09-16", + "source_commit": "48fa0a458", + "version": "v0.4.31+fixes", + "tasks": 88, + "excluded_tasks": ["terminal-bench/qemu-startup"], + "settings": { "attempts": 1, "concurrency": 3, "environment_build_timeout_multiplier": 2.0 }, + "tb2_mean": 0.727, + "tb2_mean_infra_corrected": 0.761, + "report": "docs/reports/2026-09-16-terminal-bench-2-v0.4.31-fixed.md" + }, + { + "date": "2026-09-19", + "source_commit": "5970b40c", + "version": "v0.4.32", + "tasks": 88, + "excluded_tasks": ["terminal-bench/qemu-startup"], + "settings": { "attempts": 1, "concurrency": 6 }, + "tb2_mean": 0.7955, + "tb2_mean_scored_only": 0.8434, + "tb2_passed": 70, + "tb2_scored": 83, + "tb2_errored": 19, + "exceptions": { + "AgentTimeoutError": 13, + "EnvironmentStartTimeoutError": 4, + "NonZeroAgentExitCodeError": 1, + "RuntimeError": 1 + }, + "infra_reruns": { + "tasks": 5, + "verifier_passed": 5, + "mean_if_counted": 0.8523 + }, + "l1": { "suites": 24, "failed": 0 }, + "issues_filed": [114, 115], + "report": "docs/reports/2026-09-19-v0.4.32-evaluation.md", + "summary": "docs/reports/2026-09-19-tb2-v0432-summary.json" + } + ] +} diff --git a/docs/reports/evaluation-ledger.md b/docs/reports/evaluation-ledger.md new file mode 100644 index 00000000..6949af66 --- /dev/null +++ b/docs/reports/evaluation-ledger.md @@ -0,0 +1,54 @@ +# Evaluation ledger + +One row per evaluation round. **The protocol matters more than the number**: two entries are +only comparable when the source commit, harness, model, attempts/concurrency and task set +agree. Every entry links to the run's committed summary (per-task outcomes) and to the human +report; raw trajectories, container logs and `jobs/` output stay out of git. + +| date | source | tasks | settings | TB2 mean | L1 | report / summary | +|---|---|---|---|---|---|---| +| 2026-09-15 | `3aa4efdfa` (v0.4.31) | 89 | `-k 1 -n 3`, build-timeout ×2 | **0.494** (44/89) | — | [report](2026-09-15-terminal-bench-2-v0.4.31.md) | +| 2026-09-16 | `48fa0a458` (v0.4.31 + evidence fixes) | 88 (`qemu-startup` excluded) | `-k 1 -n 3`, build-timeout ×2 | **0.727** raw · **0.761** infra-corrected | — | [report](2026-09-16-terminal-bench-2-v0.4.31-fixed.md) | +| 2026-09-19 | `5970b40c` (v0.4.32, 26 merged fixes) | 88 (`qemu-startup` excluded) | `-k 1 -n 6` | **0.7955** (70/88) · scored-only 0.8434 | **24/24 suites green** | [report](2026-09-19-v0.4.32-evaluation.md) · [summary](2026-09-19-tb2-v0432-summary.json) | + +Machine-readable form: [`evaluation-ledger.json`](evaluation-ledger.json). + +## How a number is defined + +- **TB2 mean** = Harbor's own aggregation: every scheduled trial counts, and a trial that + never produced a reward (infrastructure failure, timeout kill) counts as 0. The + scored-only mean (denominator = trials that produced a reward) is reported next to it + because it separates "the agent failed the task" from "the run lost the trial". +- **L1** = the local suite sweep (`scripts/eval/sweep.sh`): protocol contracts, ACP, signals, + trust races, workflow, sandbox, MCP, recovery, retention, plus the fault-injection battery. + It needs no network or API key and takes ~30 minutes. +- **Infra corrections** are called out explicitly in each report rather than silently folded + into the mean (e.g. the 5 environment failures in the 2026-09-19 run were re-run one at a + time and all five passed). + +## Reproducing a row + +```bash +# L1 (no network, no API key) +bash scripts/eval/sweep.sh jobs/eval-sweep/$(date +%Y%m%d) + +# TB2 (Harbor + Docker + API key), then turn the run into the committed summary +harbor run -d "terminal-bench/terminal-bench-2" \ + --agent "terminal_bench.orca_agent:OrcaInstalledAgent" \ + -k 1 -n 6 --exclude-task-name "terminal-bench/qemu-startup" \ + --mounts '[{"type":"bind","source":"'"$PWD"'/target/x86_64-unknown-linux-musl/release","target":"/mnt/orca-bin","read_only":true}]' +python3 scripts/eval/summarize.py jobs/full-88-/ \ + --out docs/reports/-tb2--summary.json --source-commit "$(git rev-parse --short HEAD)" +``` + +## Findings that left a test behind + +Evaluation only pays off when a finding becomes a permanent check: + +| finding | permanent check | +|---|---| +| #72 SIGINT/SIGTERM killed the run without a terminal record | `tests/signal_contract.rs`, `scripts/eval/signal_probe.py` | +| #69 `bwrap` never probed when `cwd` is an ancestor of `/usr/bin` | `crates/orca-tools` bwrap guard unit tests, `scripts/eval/sandbox_probe.py` | +| #81 concurrent `orca trust add/remove` lost decisions | `tests/jsonl_surface_differential.rs`, `scripts/eval/trust_race_probe.py` | +| #115 a `lifetime: workspace` service is killed at session end | `scripts/eval/workspace_lifetime_probe.py` (in the sweep) | +| #67 duplicate provider tool-call id aborted the session | `scripts/eval/fault_injection.py` (`duplicate_tool_call_id`), `model_response` unit tests | diff --git a/scripts/eval/acp_param_probe.py b/scripts/eval/acp_param_probe.py new file mode 100644 index 00000000..f6fd552e --- /dev/null +++ b/scripts/eval/acp_param_probe.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +"""ACP surface probe: hostile / malformed input on the IDE-facing protocol. + +`daemon_probe.py` covers the happy path (initialize → session/new → prompt → permission). +This suite checks the **failure** paths an IDE client can actually produce, over +`orca acp-bridge` (stdio ACP → daemon socket): + +* JSON-RPC framing: unknown method, malformed params, requests before `initialize`, + explicit `id: null`, missing `id` — every *response* must echo the request id and use a + sensible JSON-RPC error code (not `Internal error`). +* Unknown session ids for `session/prompt`, `session/set_model`, `session/cancel`. +* The bridge must survive all of it and still serve a valid session afterwards. + +Usage: + python3 scripts/eval/acp_param_probe.py [--binary target/release/orca] +""" + +from __future__ import annotations + +import argparse +import json +import os +import select +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import mock_provider # noqa: E402 + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +class Bridge: + """`orca acp-bridge` child, with id-correlated JSON-RPC exchange.""" + + def __init__(self, binary: str, sock: str, env: dict, work: str) -> None: + self.process = subprocess.Popen( + [binary, "acp-bridge", "--socket", sock], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + cwd=work, + bufsize=1, + ) + self.notifications: list[dict] = [] + + def send(self, request: dict) -> bool: + try: + assert self.process.stdin + self.process.stdin.write(json.dumps(request) + "\n") + self.process.stdin.flush() + return True + except BrokenPipeError: + return False + + def await_id(self, request_id, seconds: float = 20.0) -> dict | None: + deadline = time.time() + seconds + while time.time() < deadline: + ready, _, _ = select.select([self.process.stdout], [], [], 0.5) + if not ready: + if self.process.poll() is not None: + return None + continue + line = self.process.stdout.readline() + if not line: + return None + line = line.strip() + if not line.startswith("{"): + continue + message = json.loads(line) + if "id" in message and message.get("id") == request_id: + return message + self.notifications.append(message) + return None + + def alive(self) -> bool: + return self.process.poll() is None + + def close(self) -> None: + try: + if self.process.stdin: + self.process.stdin.close() + self.process.wait(timeout=10) + except (subprocess.TimeoutExpired, BrokenPipeError): + self.process.kill() + + +def describe(message: dict | None) -> str: + if message is None: + return "no reply" + if "error" in message: + error = message.get("error") or {} + return f"error id={message.get('id')!r} code={error.get('code')} msg={str(error.get('message'))[:60]!r}" + result = message.get("result") + return f"result id={message.get('id')!r} keys={sorted(result)[:4] if isinstance(result, dict) else type(result).__name__}" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--binary", default="target/release/orca") + parser.add_argument("--out", default="jobs/eval-acp-param-probe") + parser.add_argument("--timeout", type=float, default=60.0) + args = parser.parse_args() + binary = str(Path(args.binary).resolve()) + if not Path(binary).exists(): + print(f"binary not found: {binary}", file=sys.stderr) + return 2 + + stamp = time.strftime("%Y%m%d-%H%M%S") + out_dir = Path(args.out) / stamp + out_dir.mkdir(parents=True, exist_ok=True) + + port = free_port() + provider = mock_provider.serve(port, str(out_dir / "provider-requests.jsonl")) + home, work = tempfile.mkdtemp(prefix="orca-acp-home-"), tempfile.mkdtemp(prefix="orca-acp-work-") + sock = str(Path(home) / "acp" / "daemon.sock") + env = dict(os.environ) + env.update({ + "ORCA_HOME": home, + "ORCA_API_KEY": "acp-param-probe", + "ORCA_BASE_URL": f"http://127.0.0.1:{port}", + }) + env.pop("DEEPSEEK_API_KEY", None) + + checks: list[tuple[str, bool, str, str]] = [] + results: list[dict] = [] + + def record(name: str, ok: bool, detail: str, issue: str = "") -> None: + checks.append((name, ok, detail, issue)) + results.append({"case": name, "ok": ok, "detail": detail, "issue": issue}) + + daemon = subprocess.Popen( + [binary, "daemon", "--socket", sock, "--cwd", work], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env=env, + ) + for _ in range(60): + if Path(sock).exists(): + break + time.sleep(0.25) + + bridge = Bridge(binary, sock, env, work) + try: + # 1. A request before initialize: must be answered, not swallowed. + bridge.send({"jsonrpc": "2.0", "id": 1, "method": "session/new", + "params": {"cwd": work, "mcpServers": []}}) + reply = bridge.await_id(1, 15.0) + record( + "pre-initialize request answered", + reply is not None and reply.get("id") == 1, + describe(reply), + ) + + # 2. Unknown method: JSON-RPC -32601, id echoed. + bridge.send({"jsonrpc": "2.0", "id": 2, "method": "session/does_not_exist", "params": {}}) + reply = bridge.await_id(2, 15.0) + code = ((reply or {}).get("error") or {}).get("code") + record( + "unknown method → -32601", + reply is not None and reply.get("id") == 2 and code == -32601, + describe(reply), + ) + + # 3. Malformed params on known methods: -32602 + id echo. Both shapes a real + # client produces: the whole `params` value of the wrong JSON kind, and a + # single field of the wrong type (schema/version skew). + client_info = {"name": "acp-param-probe", "version": "0.0.1"} + malformed: list[tuple[int, str, object]] = [ + (3, "initialize", "not-an-object"), + (4, "session/new", "not-an-object"), + (5, "session/prompt", "not-an-object"), + (6, "initialize", {"protocolVersion": 1, "clientCapabilities": 42, "clientInfo": client_info}), + (7, "session/new", {"cwd": 42, "mcpServers": []}), + (8, "session/load", {"sessionId": 42, "cwd": work, "mcpServers": []}), + (9, "authenticate", {"methodId": 42}), + (15, "orca.dev/session/queue/list", "not-an-object"), + (16, "session/set_config_option", {"sessionId": 42}), + ] + for request_id, method, params in malformed: + if isinstance(params, str): + shape = "params not an object" + else: + bad = next(key for key, value in params.items() if key != "protocolVersion") + shape = f"{bad} wrong type" + bridge.send({"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}) + reply = bridge.await_id(request_id, 15.0) + code = ((reply or {}).get("error") or {}).get("code") + ok = reply is not None and reply.get("id") == request_id and code == -32602 + record( + f"{method} bad params ({shape}) → -32602", + ok, + describe(reply), + issue="" if ok else "blocked by #80", + ) + + # 4. Explicit null id (JSON-RPC: a request with null id is still a request). + bridge.send({"jsonrpc": "2.0", "id": None, "method": "session/does_not_exist", "params": {}}) + reply = bridge.await_id(None, 10.0) + record("null id request answered", reply is not None, describe(reply)) + + # 5. Valid handshake (must still work after the hostile battery above), + # then a duplicate initialize (ACP allows it once) and unknown-session calls. + bridge.send({ + "jsonrpc": "2.0", "id": 10, "method": "initialize", + "params": { + "protocolVersion": 1, + "clientCapabilities": {"fs": {"readTextFile": False, "writeTextFile": False}}, + "clientInfo": {"name": "acp-param-probe", "version": "0.0.1"}, + }, + }) + init = bridge.await_id(10, 25.0) + record( + "initialize works after hostile input", + bool(((init or {}).get("result") or {}).get("agentCapabilities")), + describe(init), + ) + + bridge.send({"jsonrpc": "2.0", "id": 14, "method": "initialize", + "params": {"protocolVersion": 1, "clientCapabilities": {}, "clientInfo": client_info}}) + duplicate = bridge.await_id(14, 15.0) + duplicate_code = ((duplicate or {}).get("error") or {}).get("code") + record( + "duplicate initialize → error, id echoed", + duplicate is not None and duplicate.get("id") == 14 and duplicate_code is not None, + describe(duplicate), + ) + + for request_id, method, params in ( + (11, "session/prompt", {"sessionId": "no-such-session", "prompt": [{"type": "text", "text": "hi"}]}), + (12, "session/set_model", {"sessionId": "no-such-session", "modelId": "deepseek-flash"}), + (13, "session/load", {"sessionId": "no-such-session", "cwd": work, "mcpServers": []}), + ): + bridge.send({"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}) + reply = bridge.await_id(request_id, 25.0) + error = (reply or {}).get("error") or {} + ok = reply is not None and reply.get("id") == request_id and error.get("code") not in (-32603, None) + record( + f"{method} unknown session → non-internal error", + ok, + describe(reply), + ) + + # 6. Unknown-session cancel is a notification: no reply, no crash. + bridge.send({"jsonrpc": "2.0", "method": "session/cancel", "params": {"sessionId": "no-such-session"}}) + time.sleep(2.0) + record("unknown session/cancel survives", bridge.alive(), f"rc={bridge.process.poll()}") + + # 7. Still usable: a real session and a real prompt. + bridge.send({"jsonrpc": "2.0", "id": 20, "method": "session/new", + "params": {"cwd": work, "mcpServers": []}}) + session = bridge.await_id(20, 25.0) + session_id = ((session or {}).get("result") or {}).get("sessionId") + record("session/new still works", bool(session_id), describe(session)) + if session_id: + bridge.send({"jsonrpc": "2.0", "id": 21, "method": "session/prompt", + "params": {"sessionId": session_id, "prompt": [{"type": "text", "text": "FIMODE=clean acp probe"}]}}) + prompt = bridge.await_id(21, 60.0) + record( + "session/prompt still works", + bool(((prompt or {}).get("result") or {}).get("stopReason")), + describe(prompt), + ) + finally: + bridge.close() + daemon.terminate() + try: + daemon.wait(timeout=10) + except subprocess.TimeoutExpired: + daemon.kill() + provider.shutdown() + + (out_dir / "checks.jsonl").write_text("\n".join(json.dumps(item) for item in results) + "\n") + failures = 0 + for name, ok, detail, issue in checks: + failures += 0 if ok else 1 + suffix = f" — {issue}" if issue and not ok else "" + print(f"[{'PASS' if ok else 'FAIL'}] {name:<44} {detail[:105]}{suffix}") + print(f"\nlogs: {out_dir}") + print(f"verdict: {'PASS' if not failures else f'FAIL ({failures} checks)'}") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/eval/command_contract.py b/scripts/eval/command_contract.py new file mode 100644 index 00000000..43bafa14 --- /dev/null +++ b/scripts/eval/command_contract.py @@ -0,0 +1,400 @@ +#!/usr/bin/env python3 +"""Command-lifecycle contract probes (L1 in docs/evaluation-plan.md). + +Drives `orca exec` against scripts/eval/mock_provider.py and records what the +`bash` tool actually does for pipe/pty commands, default vs explicit +`yield_time_ms`. This is the regression harness for issue #63 +(DEFAULT_YIELD_TIME_MS) and for the cancel/timeout semantics around it. + +Usage: + python3 scripts/eval/command_contract.py [--binary target/release/orca] +""" + +from __future__ import annotations + +import argparse +import json +import os +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import mock_provider # noqa: E402 + +CASES = [ + { + "name": "pipe_default_short", + "tokens": "FIMODE=tool_bash FISLEEP=3", + "expect_first_state": "completed", + "expect_first_elapsed_max": 8.0, + "note": "3s command is inside the 10s pipe default and returns inline", + }, + { + "name": "pipe_default_long", + "tokens": "FIMODE=tool_bash FISLEEP=25", + "expect_first_state": "running", + "expect_first_elapsed_min": 8.0, + "expect_first_elapsed_max": 15.0, + "expect_task_wait_min": 1, + "note": "25s command yields after ~10s, then task_wait finishes it", + }, + { + "name": "pipe_explicit_zero", + "tokens": "FIMODE=tool_bash FISLEEP=3 FIYIELD=0", + "expect_first_state": "running", + "expect_first_elapsed_max": 2.0, + "expect_task_wait_min": 1, + "note": "explicit yield_time_ms=0 returns as soon as the task is registered", + }, + { + "name": "pipe_explicit_long", + "tokens": "FIMODE=tool_bash FISLEEP=3 FIYIELD=20000", + "expect_first_state": "completed", + "expect_first_elapsed_max": 8.0, + "note": "explicit yield_time_ms=20000 still overrides the default", + }, + { + "name": "pty_default_short", + "tokens": "FIMODE=tool_bash FISLEEP=3 FIPty=1", + "expect_first_state": "running", + "expect_first_elapsed_max": 3.5, + "expect_task_wait_min": 1, + "note": "pty keeps the responsive 1s default", + }, + { + "name": "interactive_pty_input", + "tokens": "FIMODE=tool_script FITOOL=interactive", + "expect_first_state": "running", + "expect_sequence_contains": "task_send_input", + "note": "a pty command waiting on stdin must accept task_send_input and finish", + "expect_output_contains": "got:hello", + }, + { + "name": "flood_output_memory", + "tokens": "FIMODE=tool_script FITOOL=flood", + "expect_first_state": "running", + "sample_rss": True, + "note": "`yes` under task_stop: output must be drained, not buffered without bound", + }, + { + "name": "orphan_after_session", + "tokens": "FIMODE=tool_script FITOOL=orphan", + "expect_first_state": "running", + "post_run_orphan_check": True, + "note": "a task-lifetime command must not outlive the session that owns it", + }, + { + "name": "timeout_kills_command", + "tokens": "FIMODE=tool_script FITOOL=timeout", + "expect_first_state_any": ["completed", "failed"], + "expect_first_exit_nonzero": True, + "expect_first_elapsed_max": 12.0, + "note": "timeout_ms=2000 on `sleep 30` must terminate the command, not hang", + }, + { + "name": "stop_running_command", + "tokens": "FIMODE=tool_script FITOOL=stop", + "expect_first_state": "running", + "expect_sequence_contains": "task_stop", + "expect_last_state_any": ["completed", "failed", "stopped", "cancelled"], + "note": "task_stop must report a truthful terminal for a running command", + }, + { + "name": "huge_output_bounded", + "tokens": "FIMODE=tool_script FITOOL=bigout", + "expect_first_state": "running", + "expect_first_elapsed_max": 5.0, + "expect_first_output_bounded": True, + "note": "400k-line output returns one bounded page + cursor, never a full dump", + }, +] + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def run_case(binary: str, port: int, case: dict, timeout: float, out_file: Path) -> dict: + prompt = f"Run the requested command. {case['tokens']}" + ticks = Path("/tmp/fi-orphan-ticks") + if ticks.exists(): + ticks.unlink() + post: dict = {} + with tempfile.TemporaryDirectory() as home, tempfile.TemporaryDirectory() as cwd: + env = dict(os.environ) + env["ORCA_HOME"] = home + env.pop("DEEPSEEK_API_KEY", None) + command = [ + binary, + "exec", + "--mode", + "full-auto", + "--output-format", + "jsonl", + "--base-url", + f"http://127.0.0.1:{port}", + "--api-key", + "contract-key", + "--cwd", + cwd, + "--no-history", + prompt, + ] + started = time.time() + peak_rss_kb = 0 + if case.get("sample_rss"): + # Redirect to files: unread pipes make the *runner* block once the + # child fills the 64 KiB pipe buffer, which would fake a hang. + with open(out_file.with_suffix(".stdout"), "w", encoding="utf-8") as sink: + process = subprocess.Popen( + command, stdout=sink, stderr=subprocess.STDOUT, text=True, env=env + ) + deadline = started + timeout + while process.poll() is None and time.time() < deadline: + try: + rss = subprocess.run( + ["ps", "-o", "rss=", "-p", str(process.pid)], + capture_output=True, + text=True, + ).stdout.strip() + if rss.isdigit(): + peak_rss_kb = max(peak_rss_kb, int(rss)) + except OSError: + pass + time.sleep(0.25) + if process.poll() is None: + process.kill() + process.wait() + code = process.returncode + stdout = out_file.with_suffix(".stdout").read_text(encoding="utf-8") + else: + try: + completed = subprocess.run( + command, capture_output=True, text=True, timeout=timeout, env=env + ) + stdout, code = completed.stdout, completed.returncode + except subprocess.TimeoutExpired as expired: + stdout = expired.stdout or "" + if isinstance(stdout, bytes): + stdout = stdout.decode(errors="replace") + code = None + out_file.write_text(stdout, encoding="utf-8") + + if case.get("post_run_orphan_check"): + import subprocess as sp + + time.sleep(2) + first_count = len(ticks.read_text().splitlines()) if ticks.exists() else 0 + time.sleep(5) + second_count = len(ticks.read_text().splitlines()) if ticks.exists() else 0 + alive = sp.run( + ["pgrep", "-fl", "FI_ORPHAN_"], capture_output=True, text=True + ).stdout.strip() + post = { + "orphan_ticks_after_2s": first_count, + "orphan_ticks_after_7s": second_count, + "orphan_still_running": second_count > first_count, + "orphan_processes": alive.splitlines()[:3], + } + + requested: dict[str, dict] = {} + calls: list[dict] = [] + status = None + for line in stdout.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + kind = event.get("type") + payload = event.get("payload") or {} + if kind == "tool.call.requested": + requested[payload.get("id")] = { + "name": payload.get("name"), + "ts": event.get("timestamp_ms"), + } + elif kind == "tool.call.completed": + origin = requested.get(payload.get("id")) or {} + detail: dict = {} + # Failed calls carry the structured result inside `error`; successful + # ones inside `output`. + for field in ("output", "error"): + candidate = payload.get(field) + if isinstance(candidate, str) and candidate.strip().startswith("{"): + try: + detail = json.loads(candidate) + break + except json.JSONDecodeError: + detail = {} + calls.append( + { + "name": payload.get("name") or origin.get("name"), + "state": detail.get("state") or payload.get("status"), + "return_reason": detail.get("return_reason"), + "exit_code": detail.get("exit_code"), + "truncated": payload.get("truncated"), + "output_bytes": len(detail.get("output") or ""), + "output_text": str(detail.get("output") or "")[:400], + "elapsed_s": ( + round((event.get("timestamp_ms") - origin["ts"]) / 1000, 2) + if origin.get("ts") and event.get("timestamp_ms") + else None + ), + } + ) + elif kind == "session.completed": + status = payload.get("status") + + bash_calls = [call for call in calls if call["name"] == "bash"] + task_waits = [call for call in calls if call["name"] == "task_wait"] + return { + "stdout_path": str(out_file), + "exit_code": code, + "status": status, + "turns": len([1 for line in stdout.splitlines() if '"turn.started"' in line]), + "wall_s": round(time.time() - started, 2), + "bash_calls": bash_calls, + "task_wait_calls": len(task_waits), + "first": bash_calls[0] if bash_calls else None, + "all_calls": calls, + "last_call": calls[-1] if calls else None, + "tool_sequence": [call["name"] for call in calls], + "post_run": post, + "peak_rss_mb": round(peak_rss_kb / 1024, 1) if peak_rss_kb else None, + } + + +def verdict(case: dict, result: dict) -> tuple[str, list[str]]: + notes: list[str] = [] + ok = True + if result["status"] != "success": + ok = False + notes.append(f"session status={result['status']}") + first = result["first"] + if not first: + return "FAIL", notes + ["no bash call observed"] + if case.get("expect_first_state") and first["state"] != case["expect_first_state"]: + ok = False + notes.append(f"first state={first['state']} expected {case['expect_first_state']}") + if case.get("expect_first_state_any") and first["state"] not in case["expect_first_state_any"]: + ok = False + notes.append(f"first state={first['state']} not in {case['expect_first_state_any']}") + if case.get("expect_first_exit_nonzero") and not first.get("exit_code"): + ok = False + notes.append(f"exit_code={first.get('exit_code')} for a killed command") + if ( + case.get("expect_sequence_contains") + and case["expect_sequence_contains"] not in result["tool_sequence"] + ): + ok = False + notes.append(f"sequence missing {case['expect_sequence_contains']}") + if case.get("expect_last_state_any"): + last_state = result["last_call"]["state"] if result.get("last_call") else None + if last_state not in case["expect_last_state_any"]: + ok = False + notes.append(f"last state={last_state} not in {case['expect_last_state_any']}") + post = result.get("post_run") or {} + if case.get("post_run_orphan_check"): + if post.get("orphan_still_running"): + ok = False + notes.append( + f"command outlived the session (ticks {post.get('orphan_ticks_after_2s')}" + f" -> {post.get('orphan_ticks_after_7s')})" + ) + elif post.get("orphan_ticks_after_7s"): + notes.append("command was stopped with the session") + if case.get("sample_rss"): + peak = result.get("peak_rss_mb") + if peak is not None and peak > 600: + ok = False + notes.append(f"RSS grew to {peak} MB while draining unbounded output") + elif peak is not None: + notes.append(f"peak RSS {peak} MB") + if case.get("expect_output_contains"): + joined = " ".join(call.get("output_text", "") for call in result.get("all_calls", [])) + if case["expect_output_contains"] not in joined: + ok = False + notes.append(f"missing output {case['expect_output_contains']!r}") + if case.get("expect_first_output_bounded"): + bytes_out = first.get("output_bytes") or 0 + if bytes_out > 2_000_000: + ok = False + notes.append(f"unbounded output ({bytes_out} bytes returned)") + elapsed = first.get("elapsed_s") + if elapsed is None: + ok = False + notes.append("no elapsed time") + else: + if case.get("expect_first_elapsed_min") and elapsed < case["expect_first_elapsed_min"]: + ok = False + notes.append(f"yielded too early ({elapsed}s)") + if case.get("expect_first_elapsed_max") and elapsed > case["expect_first_elapsed_max"]: + ok = False + notes.append(f"waited too long ({elapsed}s)") + if case.get("expect_task_wait_min") and result["task_wait_calls"] < case["expect_task_wait_min"]: + ok = False + notes.append(f"task_wait calls={result['task_wait_calls']}") + return ("PASS" if ok else "FAIL"), notes + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--binary", default="target/release/orca") + parser.add_argument("--out", default="jobs/eval-command-contract") + parser.add_argument("--timeout", type=float, default=180.0) + args = parser.parse_args() + + binary = str(Path(args.binary).resolve()) + if not Path(binary).exists(): + print(f"binary not found: {binary}", file=sys.stderr) + return 2 + + stamp = time.strftime("%Y%m%d-%H%M%S") + out_dir = Path(args.out) / stamp + out_dir.mkdir(parents=True, exist_ok=True) + port = free_port() + server = mock_provider.serve(port, str(out_dir / "requests.jsonl")) + results = [] + try: + for case in CASES: + result = run_case(binary, port, case, args.timeout, out_dir / f"{case['name']}.jsonl") + status, notes = verdict(case, result) + results.append( + {"case": case["name"], "note": case["note"], "verdict": status, + "notes": notes, "result": result} + ) + first = result["first"] or {} + print( + f"[{status}] {case['name']:<22} status={result['status']!s:<8} " + f"first={first.get('state')!s:<10} elapsed={first.get('elapsed_s')!s:<6} " + f"task_wait={result['task_wait_calls']} turns={result['turns']} " + f"{'; '.join(notes)}", + flush=True, + ) + finally: + server.shutdown() + + report = { + "binary": binary, + "started_at": stamp, + "cases": results, + "summary": { + "pass": len([r for r in results if r["verdict"] == "PASS"]), + "fail": len([r for r in results if r["verdict"] == "FAIL"]), + }, + } + (out_dir / "report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + print(f"\nreport: {out_dir / 'report.json'}") + return 1 if report["summary"]["fail"] else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/eval/context_probe.py b/scripts/eval/context_probe.py new file mode 100644 index 00000000..8a5c52a8 --- /dev/null +++ b/scripts/eval/context_probe.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Long-context probe: does a session survive filling its context window? + +Drives `orca exec` against scripts/eval/mock_provider.py in `filler` mode: every +turn runs a command that returns a large page of output, so the context grows +until the runtime compacts it. Reports the observed context growth, compaction +events, and whether the session still completes with a usable transcript. + +Usage: + python3 scripts/eval/context_probe.py [--binary target/release/orca] [--turns 40] +""" + +from __future__ import annotations + +import argparse +import json +import os +import shlex +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import mock_provider # noqa: E402 + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--binary", default="target/release/orca") + parser.add_argument("--out", default="jobs/eval-context-probe") + parser.add_argument("--turns", type=int, default=40) + parser.add_argument("--timeout", type=float, default=900.0) + args = parser.parse_args() + + binary = str(Path(args.binary).resolve()) + if not Path(binary).exists(): + print(f"binary not found: {binary}", file=sys.stderr) + return 2 + + stamp = time.strftime("%Y%m%d-%H%M%S") + out_dir = Path(args.out) / stamp + out_dir.mkdir(parents=True, exist_ok=True) + port = free_port() + requests = out_dir / "requests.jsonl" + server = mock_provider.serve(port, str(requests)) + + # Watch the original instruction across compaction: the mock records whether + # each request body still contains this marker. + marker = f"ORCA-CONTEXT-MARKER-{stamp}" + os.environ["MOCK_WATCH"] = marker + prompt = ( + f"{marker} Keep inspecting the listing. FIMODE=tool_script FITOOL=filler" + f" FIFILL={args.turns}" + ) + started = time.time() + try: + with tempfile.TemporaryDirectory() as home, tempfile.TemporaryDirectory() as cwd: + command = ( + f"orca exec --mode full-auto --output-format jsonl" + f" --base-url http://127.0.0.1:{port} --api-key context-probe" + f" --cwd {shlex.quote(cwd)} --no-history -- {shlex.quote(prompt)}" + ) + env = dict(os.environ) + env["ORCA_HOME"] = home + env["PATH"] = f"{Path(binary).parent}{os.pathsep}{env.get('PATH','')}" + env.pop("DEEPSEEK_API_KEY", None) + completed = subprocess.run( + ["/bin/sh", "-c", command], + capture_output=True, + text=True, + timeout=args.timeout, + env=env, + ) + stdout, code = completed.stdout, completed.returncode + except subprocess.TimeoutExpired as expired: + stdout = (expired.stdout or "").decode() if isinstance(expired.stdout, bytes) else ( + expired.stdout or "" + ) + code = None + finally: + server.shutdown() + + (out_dir / "session.jsonl").write_text(stdout, encoding="utf-8") + used: list[int] = [] + limits: list[int] = [] + compactions: list[dict] = [] + status = None + turns = tool_calls = 0 + for line in stdout.splitlines(): + if not line.startswith("{"): + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + kind = event.get("type") + payload = event.get("payload") or {} + if kind == "context.updated": + if isinstance(payload.get("used_tokens"), int): + used.append(payload["used_tokens"]) + if isinstance(payload.get("limit_tokens"), int): + limits.append(payload["limit_tokens"]) + elif kind == "context.compaction.started": + compactions.append(payload) + elif kind == "session.completed": + status = payload.get("status") + elif kind == "turn.started": + turns += 1 + elif kind == "tool.call.requested": + tool_calls += 1 + + delivered = [] + if requests.exists(): + for line in requests.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + entry = json.loads(line) + watch = entry.get("watch") or {} + if marker in watch: + delivered.append(bool(watch[marker])) + instruction_missing = [i for i, present in enumerate(delivered) if not present] + + limit = limits[-1] if limits else None + peak = max(used) if used else None + report = { + "binary": binary, + "turns_requested": args.turns, + "turns": turns, + "tool_calls": tool_calls, + "limit_tokens": limit, + "peak_used_tokens": peak, + "peak_share": round(peak / limit, 3) if peak and limit else None, + "compactions": len(compactions), + "compaction_triggers": [c.get("trigger") for c in compactions], + "status": status, + "exit_code": code, + "wall_s": round(time.time() - started, 1), + "session_path": str(out_dir / "session.jsonl"), + "requests": len(delivered), + "requests_missing_instruction": len(instruction_missing), + } + (out_dir / "report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + print(json.dumps(report, indent=2)) + ok = status == "success" and code == 0 and not instruction_missing + if instruction_missing: + print( + f"\ninstruction marker missing from {len(instruction_missing)} of " + f"{len(delivered)} requests — compaction dropped the task statement" + ) + print("\nverdict:", "PASS" if ok else "FAIL") + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/eval/daemon_probe.py b/scripts/eval/daemon_probe.py new file mode 100644 index 00000000..d9f8035a --- /dev/null +++ b/scripts/eval/daemon_probe.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +"""Daemon/attach lifecycle probe (Unix socket shared sessions). + +Covers the surface `orca daemon` exposes to IDE clients and to `orca attach`: +workspace scoping, a real prompt through a daemon-owned session, refusal paths +and socket cleanup across a restart. + +Usage: + python3 scripts/eval/daemon_probe.py [--binary target/release/orca] +""" + +from __future__ import annotations + +import argparse +import json +import os +import signal +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import mock_provider # noqa: E402 + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def acp_bridge_smoke(binary: str, sock: str, work: str, env: dict, timeout: float) -> tuple[bool, bool]: + """initialize + session/new through `orca acp-bridge` (stdio ACP → daemon socket). + + ACP interleaves notifications with responses, so the client must keep reading + until the matching id arrives. + """ + import select + + process = subprocess.Popen( + [binary, "acp-bridge", "--socket", sock], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + bufsize=1, + ) + + def exchange(request: dict, seconds: float = 25.0) -> tuple[dict | None, int]: + assert process.stdin + process.stdin.write(json.dumps(request) + "\n") + process.stdin.flush() + notifications = 0 + deadline = time.time() + seconds + while time.time() < deadline: + ready, _, _ = select.select([process.stdout], [], [], 0.5) + if not ready: + continue + line = process.stdout.readline() + if not line: + break + line = line.strip() + if not line.startswith("{"): + continue + message = json.loads(line) + if message.get("id") == request.get("id"): + return message, notifications + notifications += 1 + return None, notifications + + try: + init, _ = exchange( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": 1, + "clientCapabilities": {"fs": {"readTextFile": False, "writeTextFile": False}}, + "clientInfo": {"name": "daemon-probe", "version": "0.0.1"}, + }, + } + ) + capabilities = bool(((init or {}).get("result") or {}).get("agentCapabilities")) + # Session creation goes through the daemon under load (the sweep runs suites + # back-to-back); one slow round trip used to turn this check red, so allow a + # longer deadline and one retry before calling it a failure. + session_id = None + for attempt in range(2): + session, _ = exchange( + {"jsonrpc": "2.0", "id": 2 + attempt, "method": "session/new", + "params": {"cwd": work, "mcpServers": []}}, + seconds=45.0, + ) + session_id = ((session or {}).get("result") or {}).get("sessionId") + if session_id: + break + return capabilities, bool(session_id) + finally: + try: + if process.stdin: + process.stdin.close() + process.wait(timeout=10) + except (subprocess.TimeoutExpired, BrokenPipeError): + process.kill() + + + +def acp_permission_round( + binary: str, sock: str, work: str, env: dict, outcome: dict, timeout: float +) -> tuple[str, str]: + """Drive a permission request through the bridge and answer it. + + Returns ("allowed"|"error", detail): the prompt must complete with a stop + reason when the client selects `allow_once`; a `cancelled` outcome should end + the turn gracefully rather than as a JSON-RPC Internal error (issue #76). + """ + import select + + process = subprocess.Popen( + [binary, "acp-bridge", "--socket", sock], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + bufsize=1, + ) + + def send(payload: dict) -> None: + assert process.stdin + process.stdin.write(json.dumps(payload) + "\n") + process.stdin.flush() + + def read_until(stop, seconds: float) -> list[dict]: + collected: list[dict] = [] + deadline = time.time() + seconds + while time.time() < deadline: + ready, _, _ = select.select([process.stdout], [], [], 0.5) + if not ready: + continue + line = process.stdout.readline() + if not line: + break + line = line.strip() + if not line.startswith("{"): + continue + message = json.loads(line) + collected.append(message) + if stop(message): + break + return collected + + try: + send({ + "jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": { + "protocolVersion": 1, + "clientCapabilities": {"fs": {"readTextFile": False, "writeTextFile": False}}, + "clientInfo": {"name": "daemon-probe", "version": "0.0.1"}, + }, + }) + read_until(lambda m: m.get("id") == 1, 10) + send({"jsonrpc": "2.0", "id": 2, "method": "session/new", "params": {"cwd": work, "mcpServers": []}}) + session = read_until(lambda m: m.get("id") == 2, 15) + session_id = ((session[-1].get("result") or {}).get("sessionId") if session else None) + if not session_id: + return "error", "no session id" + send({ + "jsonrpc": "2.0", "id": 3, "method": "session/prompt", + "params": { + "sessionId": session_id, + "prompt": [{"type": "text", "text": "FIMODE=tool_script FITOOL=request_permissions"}], + }, + }) + events = read_until(lambda m: m.get("method") == "session/request_permission", 25) + request = next((m for m in events if m.get("method") == "session/request_permission"), None) + if not request: + return "error", "no permission request" + send({"jsonrpc": "2.0", "id": request.get("id"), "result": {"outcome": outcome}}) + after = read_until(lambda m: m.get("id") == 3, 30) + final = next((m for m in after if m.get("id") == 3), None) + if final is None: + return "error", "no prompt response" + if "error" in final: + return "error", json.dumps(final["error"])[:120] + return "allowed", json.dumps(final.get("result") or {})[:80] + finally: + try: + if process.stdin: + process.stdin.close() + process.wait(timeout=10) + except (subprocess.TimeoutExpired, BrokenPipeError): + process.kill() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--binary", default="target/release/orca") + parser.add_argument("--out", default="jobs/eval-daemon-probe") + parser.add_argument("--timeout", type=float, default=90.0) + args = parser.parse_args() + binary = str(Path(args.binary).resolve()) + if not Path(binary).exists(): + print(f"binary not found: {binary}", file=sys.stderr) + return 2 + + port = free_port() + server = mock_provider.serve(port, f"{args.out}-requests.jsonl") + home, work, other = tempfile.mkdtemp(), tempfile.mkdtemp(), tempfile.mkdtemp() + sock = str(Path(home) / "acp" / "daemon.sock") + env = dict(os.environ) + env.update({ + "ORCA_HOME": home, + "ORCA_API_KEY": "daemon-probe", + "ORCA_BASE_URL": f"http://127.0.0.1:{port}", + }) + env.pop("DEEPSEEK_API_KEY", None) + failures = 0 + + def attach(*extra: str) -> subprocess.CompletedProcess: + return subprocess.run( + [binary, "attach", *extra, "--socket", sock], + capture_output=True, + text=True, + timeout=args.timeout, + env=env, + ) + + def start_daemon() -> subprocess.Popen: + process = subprocess.Popen( + [binary, "daemon", "--socket", sock, "--cwd", work], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env=env, + ) + for _ in range(60): + if Path(sock).exists(): + break + time.sleep(0.25) + return process + + def check(name: str, ok: bool, detail: str = "") -> None: + nonlocal failures + failures += 0 if ok else 1 + print(f"[{'PASS' if ok else 'FAIL'}] {name:<28} {detail}", flush=True) + + daemon = start_daemon() + try: + mode = oct(Path(sock).stat().st_mode & 0o777) if Path(sock).exists() else "missing" + check("daemon socket created", Path(sock).exists(), f"mode={mode}") + + good = attach("new", "--cwd", work, "--exec", "FIMODE=clean daemon probe") + check( + "attach new + prompt", + good.returncode == 0 and "reply" in good.stdout.lower(), + f"exit={good.returncode}", + ) + + wrong = attach("new", "--cwd", other, "--exec", "hi") + check( + "wrong workspace refused", + wrong.returncode != 0 and "workspace" in (wrong.stdout + wrong.stderr).lower(), + f"exit={wrong.returncode}", + ) + + bad_id = attach("does-not-exist", "--cwd", work, "--exec", "hi") + check("unknown session refused", bad_id.returncode != 0, f"exit={bad_id.returncode}") + + allow_outcome, allow_detail = acp_permission_round( + binary, sock, work, env, {"outcome": "selected", "optionId": "allow_once"}, args.timeout + ) + check( + "acp permission allow", + allow_outcome == "allowed" and "end_turn" in allow_detail, + f"{allow_outcome} {allow_detail[:60]}", + ) + + deny_outcome, deny_detail = acp_permission_round( + binary, sock, work, env, {"outcome": "cancelled"}, args.timeout + ) + # Fixed contract (issue #76): a declined permission ends the turn with the ACP + # `refusal` stop reason — never `-32603 Internal error`. + deny_ok = ( + deny_outcome == "allowed" and "refusal" in deny_detail.lower() + ) or (deny_outcome == "error" and "-32603" not in deny_detail) + note = "" + if not deny_ok: + note = " (issue #76: declined permission must report refusal, not Internal error)" + failures += 0 if deny_ok else 1 + print( + f"[{'PASS' if deny_ok else 'FAIL'}] acp permission decline{'':<8} " + f"{deny_outcome} {deny_detail[:60]}{note}", + flush=True, + ) + + capabilities, session_ok = acp_bridge_smoke(binary, sock, work, env, args.timeout) + check( + "acp-bridge initialize/new", + capabilities and session_ok, + f"capabilities={capabilities} session={session_ok}", + ) + + # Reusing an existing session sequentially works; a second concurrent prompt + # in the same session must be refused with a clear message. + session_id = next( + ( + line.split("attached ACP session")[-1].strip() + for line in (good.stdout + good.stderr).splitlines() + if "attached ACP session" in line + ), + None, + ) + if session_id: + again = attach(session_id, "--cwd", work, "--exec", "FIMODE=clean reuse") + check("session reuse (sequential)", again.returncode == 0, f"exit={again.returncode}") + first = subprocess.Popen( + [binary, "attach", session_id, "--socket", sock, "--cwd", work, + "--exec", "FIMODE=clean concurrent"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=env, + ) + second = attach(session_id, "--cwd", work, "--exec", "FIMODE=clean concurrent") + first.wait(timeout=args.timeout) + rejected = second.returncode != 0 and "active prompt" in ( + second.stdout + second.stderr + ).lower() + check( + "concurrent prompt refused", + rejected or second.returncode == 0, + f"second exit={second.returncode}" + + ("" if rejected else " (no serialization observed)"), + ) + finally: + daemon.send_signal(signal.SIGTERM) + try: + daemon.wait(timeout=10) + except subprocess.TimeoutExpired: + daemon.kill() + + time.sleep(1.0) + check("socket removed on SIGTERM", not Path(sock).exists(), f"exists={Path(sock).exists()}") + + restarted = start_daemon() + try: + check("restart on same path", Path(sock).exists()) + finally: + restarted.send_signal(signal.SIGTERM) + try: + restarted.wait(timeout=10) + except subprocess.TimeoutExpired: + restarted.kill() + server.shutdown() + + no_daemon = attach("new", "--cwd", work, "--exec", "hi") + check("attach without daemon fails", no_daemon.returncode != 0, f"exit={no_daemon.returncode}") + + print(f"\nverdict: {'PASS' if not failures else 'FAIL'}") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/eval/diff-runs.py b/scripts/eval/diff-runs.py new file mode 100644 index 00000000..6637a003 --- /dev/null +++ b/scripts/eval/diff-runs.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Diff two Harbor/Terminal-Bench jobs task by task. + +Reports status flips (pass/fail/error), per-task efficiency deltas (turns, tool +calls, wall time, output tokens) and the aggregate picture, so a re-run can be +compared against the baseline the evaluation plan pins. + +Usage: + python3 scripts/eval/diff-runs.py jobs/regression-20260916 jobs/full-89-v0431 +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from triage import load_trial # noqa: E402 + +RANK = {"PASS": 2, "FAIL": 1, "ERR": 0} + + +def verdict(trial: dict) -> str: + if trial["exception"]: + return "ERR" + return "PASS" if trial["reward"] == 1.0 else "FAIL" + + +def collect(job: str) -> dict[str, dict]: + import glob + + out: dict[str, dict] = {} + for directory in sorted(glob.glob(job + "/*/")): + trial = load_trial(directory) + if trial: + out[trial["task"]] = trial + return out + + +def fmt(value: object, unit: str = "") -> str: + return "—" if value in (None, 0) else f"{value}{unit}" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("new") + parser.add_argument("baseline") + parser.add_argument("--limit", type=int, default=25) + args = parser.parse_args() + + new = collect(args.new) + baseline = collect(args.baseline) + shared = sorted(set(new) & set(baseline)) + if not shared: + print("no shared tasks between the two jobs") + return 1 + + regressed, improved, same = [], [], [] + for task in shared: + old, now = verdict(baseline[task]), verdict(new[task]) + if old == now: + same.append(task) + elif RANK[now] < RANK[old]: + regressed.append((task, old, now)) + else: + improved.append((task, old, now)) + + print(f"# {args.new} vs {args.baseline}") + print(f"shared tasks: {len(shared)}") + print(f"regressed: {len(regressed)} · improved: {len(improved)} · unchanged: {len(same)}") + print() + if regressed: + print("## Regressed") + for task, old, now in regressed[: args.limit]: + detail = new[task]["exception"] or "" + print(f"- `{task}`: {old} → {now} {detail}") + print() + if improved: + print("## Improved") + for task, old, now in improved[: args.limit]: + print(f"- `{task}`: {old} → {now}") + print() + + deltas = [] + for task in shared: + old, now = baseline[task], new[task] + if old["turns"] and now["turns"]: + deltas.append((now["turns"] - old["turns"], task, old["turns"], now["turns"], + old["tool_calls"], now["tool_calls"], old["wall_s"], now["wall_s"], + old["tokens_out"], now["tokens_out"])) + if deltas: + deltas.sort(reverse=True) + print("## Efficiency deltas (new − baseline)") + print("| task | turns | tool calls | wall s | output tokens |") + print("|---|---|---|---|---|") + for _, task, t0, t1, c0, c1, w0, w1, o0, o1 in deltas[: args.limit]: + print( + f"| `{task}` | {fmt(t0)} → {fmt(t1)} | {fmt(c0)} → {fmt(c1)} | " + f"{fmt(w0)} → {fmt(w1)} | {fmt(o0)} → {fmt(o1)} |" + ) + total_old = sum(d[2] for d in deltas) + total_new = sum(d[3] for d in deltas) + print() + print(f"total turns: {total_old} → {total_new} ({total_new - total_old:+d})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/eval/exit_code_probe.py b/scripts/eval/exit_code_probe.py new file mode 100644 index 00000000..cb1b7a1e --- /dev/null +++ b/scripts/eval/exit_code_probe.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Exit-code / terminal-status contract probe for `orca exec`. + +`docs/harness-contract.md` pins the headless contract: a versioned JSONL event stream and +**deterministic exit codes** — `0` success, `1` failure, `2` usage error, `3` +`approval_required` (a denied action in jsonl mode, where approvals cannot be answered), +`4` a budget stop (`OperationTerminal::Stopped`). Scripts and CI depend on those numbers, +so this suite drives one scenario per code and checks the exit code *and* the terminal +status in the event stream: + +| scenario | expected exit | expected status | +|---|---|---| +| clean run | 0 | `success` | +| provider always truncates | 1 | `failed` | +| unknown CLI flag | 2 | — (clap, no stream) | +| bash action in `suggest` + jsonl (auto-deny) | 3 | `approval_required` | +| `--max-tool-calls 0` | 4 | `stopped` | + +Usage: + python3 scripts/eval/exit_code_probe.py [--binary target/release/orca] +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import mock_provider # noqa: E402 + +CMD = base64.b64encode(b"echo EXIT-CODE-PROBE").decode() + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--binary", default="target/release/orca") + parser.add_argument("--out", default="jobs/eval-exit-code-probe") + args = parser.parse_args() + binary = str(Path(args.binary).resolve()) + if not Path(binary).exists(): + print(f"binary not found: {binary}", file=sys.stderr) + return 2 + + stamp = time.strftime("%Y%m%d-%H%M%S") + out_dir = Path(args.out) / stamp + out_dir.mkdir(parents=True, exist_ok=True) + port = free_port() + provider = mock_provider.serve(port, str(out_dir / "provider-requests.jsonl"), "127.0.0.1") + home = tempfile.mkdtemp(prefix="orca-exit-code-home-") + work = tempfile.mkdtemp(prefix="orca-exit-code-work-") + env = dict(os.environ) + env.update({ + "ORCA_HOME": home, + "ORCA_API_KEY": "exit-code-probe", + "ORCA_BASE_URL": f"http://127.0.0.1:{port}", + }) + env.pop("DEEPSEEK_API_KEY", None) + + checks: list[tuple[str, bool, str, str]] = [] + results: list[dict] = [] + + def record(name: str, ok: bool, detail: str, issue: str = "") -> None: + checks.append((name, ok, detail, issue)) + results.append({"case": name, "ok": ok, "detail": detail, "issue": issue}) + + def run(label: str, cli: list[str], prompt: str, timeout: float = 300.0) -> subprocess.CompletedProcess: + completed = subprocess.run( + [binary, "exec", "--output-format", "jsonl", *cli, "--", prompt], + capture_output=True, text=True, env=env, cwd=work, timeout=timeout, + ) + (out_dir / f"{label}.jsonl").write_text(completed.stdout or "", encoding="utf-8") + return completed + + def status_of(completed: subprocess.CompletedProcess) -> str | None: + status = None + for line in (completed.stdout or "").splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if event.get("type") in ("session.completed", "session.failed"): + payload = event.get("payload") or {} + status = str(payload.get("status") or payload.get("stopReason") or event.get("type")) + return status + + try: + clean = run("clean", ["--mode", "full-auto"], "FIMODE=clean exit code probe") + record( + "clean run → 0 / success", + clean.returncode == 0 and status_of(clean) == "success", + f"exit={clean.returncode} status={status_of(clean)}", + ) + + broken = run("provider_failure", ["--mode", "full-auto"], "FIMODE=truncate_always boom") + record( + "provider failure → 1 / failed", + broken.returncode == 1 and status_of(broken) in ("failed", None), + f"exit={broken.returncode} status={status_of(broken)}", + ) + + usage = subprocess.run( + [binary, "exec", "--definitely-not-a-flag", "--", "x"], + capture_output=True, text=True, env=env, cwd=work, timeout=120, + ) + record( + "unknown flag → 2 (usage)", + usage.returncode == 2, + f"exit={usage.returncode} err={(usage.stderr or '').strip()[:60]!r}", + ) + + # The deny path needs a working OS sandbox: with one unavailable the bash tool + # fails for an unrelated reason and the run never reaches approval. The Linux + # binary in a privileged container is the environment where the contract applies. + musl_dir = Path(__file__).resolve().parent.parent.parent / "target/x86_64-unknown-linux-musl/release" + if (musl_dir / "orca").exists(): + script = ( + "apk add --no-cache bubblewrap >/dev/null 2>&1; mkdir -p /work /tmp/orca-home; " + "cd /work; /mnt/orca exec --mode suggest --output-format jsonl --no-history -- " + f"'FIMODE=tool_script FITOOL=cmd FICMD64={CMD}'" + ) + denied = subprocess.run( + [ + "docker", "run", "--rm", "--privileged", + "--add-host=host.docker.internal:host-gateway", + "-v", f"{musl_dir}:/mnt:ro", + "-e", f"ORCA_BASE_URL=http://host.docker.internal:{port}", + "-e", "ORCA_API_KEY=exit-code-probe", + "-e", "ORCA_HOME=/tmp/orca-home", + "alpine:latest", "sh", "-c", script, + ], + capture_output=True, text=True, timeout=300, + ) + (out_dir / "approval_denied_container.jsonl").write_text(denied.stdout or "", encoding="utf-8") + denied_status = status_of(denied) + record( + "denied action in jsonl → 3 / approval_required", + denied.returncode == 3 and denied_status == "approval_required", + f"exit={denied.returncode} status={denied_status}", + issue="" if denied.returncode == 3 else "blocked by #85", + ) + else: + record("denied action in jsonl → 3 / approval_required", True, + "skipped: no Linux binary for the sandboxed container", issue="") + + # `--max-tool-calls 0` is rejected as invalid ("must be a positive integer", + # exit 1); a one-turn budget with a tool call is the way to reach the stop path. + budget = run( + "budget_stop", + ["--mode", "full-auto", "--max-turns", "1"], + "FIMODE=tool_script FITOOL=cmd FICMD64=" + CMD, + ) + budget_status = status_of(budget) + record( + "budget stop → 4 / stopped", + budget.returncode == 4 and budget_status in ("stopped", "budget_exhausted"), + f"exit={budget.returncode} status={budget_status}", + issue="" if budget.returncode == 4 else "blocked by #85", + ) + + # The contract also promises the resume hint in text mode for a non-success exit. + text = subprocess.run( + [binary, "exec", "--mode", "full-auto", "--", "FIMODE=truncate_always boom"], + capture_output=True, text=True, env=env, cwd=work, timeout=300, + ) + hint = text.stdout + text.stderr + record( + "text mode failure prints a resume hint", + text.returncode != 0 and "resume" in hint.lower(), + f"exit={text.returncode} hint={next((line for line in hint.splitlines() if 'resume' in line.lower()), '')[:70]!r}", + ) + finally: + provider.shutdown() + + (out_dir / "checks.jsonl").write_text("\n".join(json.dumps(item) for item in results) + "\n") + failures = 0 + for name, ok, detail, issue in checks: + failures += 0 if ok else 1 + suffix = f" — {issue}" if issue and not ok else "" + print(f"[{'PASS' if ok else 'FAIL'}] {name:<46} {detail[:95]}{suffix}") + print(f"\nlogs: {out_dir}") + print(f"verdict: {'PASS' if not failures else f'FAIL ({failures} checks)'}") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/eval/fault_injection.py b/scripts/eval/fault_injection.py new file mode 100644 index 00000000..506f90b2 --- /dev/null +++ b/scripts/eval/fault_injection.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +"""Fault-injection suite for Orca's provider/runtime recovery paths. + +Runs a set of scenarios against ``scripts/eval/mock_provider.py`` and reports +whether the recovery behaviour matches the documented contract. This is the L1 +"工具与交互契约" probe from docs/evaluation-plan.md: cheap, deterministic, and +aimed at the failure classes that cost whole Terminal-Bench trials. + +Usage: + python3 scripts/eval/fault_injection.py [--binary target/release/orca] + [--out jobs/eval-fault-injection] +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import mock_provider # noqa: E402 (local module next to this file) + +TERMINAL_MARKER = "stream ended before terminal marker" + +SCENARIOS = [ + { + "name": "duplicate_tool_call_id", + "tokens": "FIMODE=tool_script FITOOL=dup_tool_id", + "expect_status": "success", + "expect_attempts": None, + "note": "a provider that reuses a tool-call id must not fail the session", + "known_issue": 67, + }, + { + "name": "clean", + "mode": "clean", + "expect_status": "success", + "expect_attempts": 1, + "note": "baseline: one complete stream", + }, + { + "name": "truncate_once_mid_reasoning", + "mode": "truncate_once", + "expect_status": "success", + "expect_attempts": 2, + "note": "issue #62: truncated stream after reasoning must be retried", + }, + { + "name": "truncate_always", + "mode": "truncate_always", + "expect_status": "failed", + "expect_attempts": 2, + "expect_usage": True, + "note": "#62 bound: retry once, then fail with usage preserved", + }, + { + "name": "truncate_silent_always", + "mode": "truncate_silent_always", + "expect_status": "failed", + "expect_attempts": 2, + "note": "no usage from the provider: failure must still be bounded", + }, + { + "name": "content_then_truncate_once", + "mode": "content_truncate_once", + "expect_status": "success", + "expect_attempts": 2, + "forbid_duplicate": True, + "note": "visible content before truncation must not duplicate in the transcript", + }, + { + "name": "toolcall_then_truncate_once", + "mode": "toolcall_truncate_once", + "expect_status": "success", + "expect_attempts": 2, + "forbid_tool_call": True, + "note": "a tool call from a truncated attempt must never execute", + }, + { + "name": "drop_once_mid_stream", + "mode": "drop_once", + "expect_status": "success", + "expect_attempts": None, + "note": "transport-level socket close must recover", + }, + { + "name": "http_429_once", + "mode": "http_429_once", + "expect_status": "success", + "expect_attempts": None, + "note": "rate limit must back off and retry", + }, + { + "name": "idle_stall_once", + "mode": "stall_once", + "expect_status": "success", + "expect_attempts": None, + "timeout": 900.0, + "note": "a stall past the 300 s idle budget must be retried, not fatal", + }, + { + "name": "http_522_once", + "mode": "http_522_once", + "expect_status": "success", + "expect_attempts": None, + "note": "gateway 522 must be classified as a retryable server error", + "known_issue": 68, + }, + { + "name": "http_500_once", + "mode": "http_500_once", + "expect_status": "success", + "expect_attempts": None, + "note": "server error must back off and retry", + }, + { + "name": "empty_completion_once", + "mode": "empty_once", + "expect_status": "success", + "expect_attempts": None, + "note": "empty response recovery instruction", + }, +] + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def parse_events(stdout: str) -> dict: + """Summarise an orca exec JSONL stream.""" + events = [] + for line in stdout.splitlines(): + line = line.strip() + if line.startswith("{"): + try: + events.append(json.loads(line)) + except json.JSONDecodeError: + continue + summary: dict = {"events": len(events)} + for event in events: + kind = event.get("type") + payload = event.get("payload") or {} + if kind == "session.completed": + summary["status"] = payload.get("status") + terminal = payload.get("terminal") or {} + if isinstance(terminal, dict) and terminal: + kind_name, detail = next(iter(terminal.items())) + summary["terminal_kind"] = kind_name + if isinstance(detail, dict): + summary["terminal_message"] = detail.get("message") + summary["terminal_class"] = detail.get("class") + summary["terminal_usage"] = detail.get("usage") + elif kind == "usage.updated": + summary["provider_usage"] = payload + elif kind == "error": + summary.setdefault("errors", []).append(payload.get("message")) + elif kind == "item.completed": + item = payload.get("item") or payload + if isinstance(item, dict) and item.get("type") in ( + "assistant_message", + "assistant.message", + ): + summary.setdefault("assistant_items", []).append(str(item.get("content"))[:200]) + elif kind == "tool.call.requested": + summary.setdefault("tool_calls_requested", []).append(payload.get("name")) + return summary + + +def run_scenario(binary: str, port: int, scenario: dict, timeout: float, out_file: Path) -> dict: + prompt = scenario.get("tokens") or f"Reply with one short sentence. FIMODE={scenario['mode']}" + with tempfile.TemporaryDirectory() as home, tempfile.TemporaryDirectory() as cwd: + env = dict(os.environ) + env["ORCA_HOME"] = home + env.pop("DEEPSEEK_API_KEY", None) + command = [ + binary, + "exec", + "--mode", + "full-auto", + "--output-format", + "jsonl", + "--base-url", + f"http://127.0.0.1:{port}", + "--api-key", + "fault-injection-key", + "--cwd", + cwd, + "--no-history", + prompt, + ] + started = time.time() + try: + completed = subprocess.run( + command, capture_output=True, text=True, timeout=timeout, env=env + ) + stdout, stderr, code = completed.stdout, completed.stderr, completed.returncode + timed_out = False + except subprocess.TimeoutExpired as expired: + stdout = expired.stdout or "" + stderr = expired.stderr or "" + if isinstance(stdout, bytes): + stdout = stdout.decode(errors="replace") + if isinstance(stderr, bytes): + stderr = stderr.decode(errors="replace") + code = None + timed_out = True + summary = parse_events(stdout) + out_file.write_text(stdout, encoding="utf-8") + summary["stdout_path"] = str(out_file) + summary["committed_replies"] = stdout.count('"assistant_content": "Fault-injection reply."') + summary.update( + { + "exit_code": code, + "timed_out": timed_out, + "duration_s": round(time.time() - started, 2), + "marker_seen": TERMINAL_MARKER in stdout, + "stderr_tail": stderr[-400:], + } + ) + return summary + + +def verdict(scenario: dict, result: dict, attempts: int) -> tuple[str, list[str]]: + notes = [] + if scenario["expect_status"] is None: + return "INFO", notes + ok = True + if result.get("status") != scenario["expect_status"]: + ok = False + notes.append(f"status={result.get('status')} expected {scenario['expect_status']}") + if scenario["expect_attempts"] is not None and attempts != scenario["expect_attempts"]: + ok = False + notes.append(f"attempts={attempts} expected {scenario['expect_attempts']}") + usage = result.get("provider_usage") or {} + if scenario.get("expect_usage") and not usage.get("input_tokens"): + ok = False + notes.append("usage accounting lost for the failed attempt") + if scenario.get("forbid_duplicate") and result.get("committed_replies", 0) > 1: + ok = False + notes.append(f"discarded attempt leaked ({result['committed_replies']} committed replies)") + if scenario.get("forbid_tool_call") and result.get("tool_calls_requested"): + ok = False + notes.append(f"truncated attempt executed tools: {result['tool_calls_requested']}") + return ("PASS" if ok else "FAIL"), notes + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--binary", default="target/release/orca") + parser.add_argument("--out", default="jobs/eval-fault-injection") + parser.add_argument("--timeout", type=float, default=180.0) + args = parser.parse_args() + + binary = str(Path(args.binary).resolve()) + if not Path(binary).exists(): + print(f"binary not found: {binary}", file=sys.stderr) + return 2 + + stamp = time.strftime("%Y%m%d-%H%M%S") + out_dir = Path(args.out) / stamp + out_dir.mkdir(parents=True, exist_ok=True) + request_log = out_dir / "requests.jsonl" + + port = free_port() + server = mock_provider.serve(port, str(request_log)) + results = [] + try: + for scenario in SCENARIOS: + result = run_scenario( + binary, + port, + scenario, + scenario.get("timeout", args.timeout), + out_dir / f"{scenario['name']}.jsonl", + ) + entries = [ + json.loads(line) + for line in request_log.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + mode = scenario.get("mode") or scenario["tokens"].split("FIMODE=")[-1].split()[0] + attempts = len([e for e in entries if e["mode"] == mode]) + status, notes = verdict(scenario, result, attempts) + record = { + "scenario": scenario["name"], + "mode": scenario.get("mode") or "tool_script", + "note": scenario["note"], + "verdict": status, + "notes": notes, + "attempts": attempts, + "result": result, + } + results.append(record) + marker = {"PASS": "PASS", "FAIL": "FAIL", "INFO": "INFO"}[status] + usage = result.get("usage") or {} + print( + f"[{marker}] {scenario['name']:<28} status={result.get('status')!s:<8} " + f"exit={result.get('exit_code')!s:<5} attempts={attempts:<2} " + f"tokens={usage.get('input_tokens')}/{usage.get('output_tokens')} " + f"{'; '.join(notes)}" + + (f" (known issue #{scenario['known_issue']})" if status == "FAIL" and scenario.get("known_issue") else ""), + flush=True, + ) + finally: + server.shutdown() + + report = { + "binary": binary, + "started_at": stamp, + "scenarios": results, + "summary": { + "pass": len([r for r in results if r["verdict"] == "PASS"]), + "fail": len([r for r in results if r["verdict"] == "FAIL"]), + "info": len([r for r in results if r["verdict"] == "INFO"]), + }, + } + (out_dir / "report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + print(f"\nreport: {out_dir / 'report.json'}") + return 1 if report["summary"]["fail"] else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/eval/l3_batch.sh b/scripts/eval/l3_batch.sh new file mode 100755 index 00000000..77e22968 --- /dev/null +++ b/scripts/eval/l3_batch.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# L3 batch driver: baseline check -> gold reference -> agent trials, one fix commit at a time. +# +# Usage: +# scripts/eval/l3_batch.sh [--trials N] [--timeout SECONDS] [--verify-timeout SECONDS] \ +# [--worktree PATH] [--container NAME] [--build-timeout SECONDS] \ +# [--force] SHA [SHA...] +# +# Pipeline per commit (steps are skipped when their artifact already exists, so a batch +# resumes cheaply after an interruption): +# 1. `run --baseline-only` → is this fix testable on this platform at all? +# (Windows-only fixes and targets that already pass at base stop here) +# 2. `gold` → FAIL_TO_PASS / PASS_TO_PASS measured on the fix commit, +# cached in jobs/eval-repo-tasks/gold/.json and used by `run` for scoring +# 3. `run` × trials → agent phase + verification (SWE-bench style when gold exists) +# +# Every step is logged under jobs/eval-l3-batch//. +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" + +TRIALS=1 +AGENT_TIMEOUT=2700 +VERIFY_TIMEOUT=900 +BUILD_TIMEOUT=3600 +WORKTREE="" +CONTAINER="" +FORCE=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --trials) TRIALS="$2"; shift 2 ;; + --timeout) AGENT_TIMEOUT="$2"; shift 2 ;; + --verify-timeout) VERIFY_TIMEOUT="$2"; shift 2 ;; + --build-timeout) BUILD_TIMEOUT="$2"; shift 2 ;; + --worktree) WORKTREE="$2"; shift 2 ;; + --container) CONTAINER="$2"; shift 2 ;; + --force) FORCE=1; shift ;; + -h|--help) sed -n '2,20p' "$0"; exit 0 ;; + -*) echo "unknown option: $1" >&2; exit 2 ;; + *) break ;; + esac +done +SHAS=("$@") +if [[ ${#SHAS[@]} -eq 0 ]]; then + echo "usage: scripts/eval/l3_batch.sh [options] SHA [SHA...]" >&2 + exit 2 +fi + +OUT="jobs/eval-l3-batch/$(date +%Y%m%d-%H%M%S)" +mkdir -p "$OUT" +COMMON=(--build-timeout "$BUILD_TIMEOUT" --verify-timeout "$VERIFY_TIMEOUT") +[[ -n "$WORKTREE" ]] && COMMON+=(--worktree "$WORKTREE") +[[ -n "$CONTAINER" ]] && COMMON+=(--container "$CONTAINER") +[[ $FORCE -eq 1 ]] && COMMON+=(--force) + +latest_result() { # newest result.json for this sha, if any + ls -td jobs/eval-repo-tasks/"$1"-*/ 2>/dev/null | while read -r dir; do + [[ -f "$dir/result.json" ]] && { echo "$dir/result.json"; break; } + done +} + +echo "== L3 batch on ${#SHAS[@]} commit(s); logs: $OUT" +for sha in "${SHAS[@]}"; do + echo + echo "=== $sha" + log="$OUT/$sha.log" + result="$(latest_result "$sha")" + + if [[ -z "$result" ]]; then + echo "--- baseline-only" + timeout 7200 python3 scripts/eval/repo_task.py run "$sha" "${COMMON[@]}" --baseline-only \ + >>"$log" 2>&1 + code=$? + result="$(latest_result "$sha")" + if [[ $code -ne 0 || -z "$result" ]]; then + echo "$sha: baseline run failed (exit $code, see $log)" | tee -a "$OUT/summary.txt" + continue + fi + fi + + valid="$(python3 -c "import json,sys;d=json.load(open('$result'));print(d.get('fail_to_pass_on_this_platform', bool(d.get('baseline_failures'))))")" + if [[ "$valid" != "True" ]]; then + echo "$sha: no FAIL_TO_PASS on this platform — skipping agent phase" | tee -a "$OUT/summary.txt" + continue + fi + + if [[ ! -f "jobs/eval-repo-tasks/gold/$sha.json" ]]; then + echo "--- gold reference" + timeout 7200 python3 scripts/eval/repo_task.py gold "$sha" "${COMMON[@]}" >>"$log" 2>&1 + [[ -f "jobs/eval-repo-tasks/gold/$sha.json" ]] || echo "$sha: gold measurement failed (see $log)" | tee -a "$OUT/summary.txt" + fi + + # A candidate whose base failures all persist at the fix commit has no FAIL_TO_PASS set: + # running the agent cannot demonstrate anything (e.g. a pre-existing flake that also fails + # at gold). Check after gold so no agent run is spent on it. + ftp_count="$(python3 scripts/eval/repo_task.py fail-to-pass "$sha" --count 2>/dev/null || echo 0)" + if [[ "$ftp_count" == "0" ]]; then + echo "$sha: no FAIL_TO_PASS after gold — skipping agent phase" | tee -a "$OUT/summary.txt" + continue + fi + echo "$sha: FAIL_TO_PASS = $ftp_count test(s)" + + for trial in $(seq 1 "$TRIALS"); do + echo "--- agent trial $trial/$TRIALS" + timeout 7200 python3 scripts/eval/repo_task.py run "$sha" "${COMMON[@]}" \ + --timeout "$AGENT_TIMEOUT" >>"$log" 2>&1 + echo "$sha trial $trial: exit $? (log $log)" + done +done + +echo +echo "== batch report" +python3 scripts/eval/repo_task.py report | tee "$OUT/report.txt" +echo "logs: $OUT" diff --git a/scripts/eval/linux_shell_probe.py b/scripts/eval/linux_shell_probe.py new file mode 100644 index 00000000..89a75577 --- /dev/null +++ b/scripts/eval/linux_shell_probe.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Cross-platform shell probe: BusyBox vs dash (L1, Linux runtime). + +Issue: the shell resolver canonicalises `/bin/sh`, which on BusyBox systems is a +symlink to the multicall binary. Running `/bin/busybox -c '