diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f96079..69cc8d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,3 +41,19 @@ jobs: run: uv run --frozen pytest - name: Test the httpx2 SDK transport run: uv run --with anthropic==1.5.0 pytest + + harbor: + name: Harbor workflow contracts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + python-version: "3.13" + - name: Test cache integrity, preflight, and verifier retries + run: >- + uv run --frozen --project benchmarks/harbor pytest + -c benchmarks/harbor/pyproject.toml benchmarks/harbor/tests diff --git a/AGENTS.md b/AGENTS.md index 492f57d..ce1cd67 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,6 +19,24 @@ the matching one: See `docs/RELEASING.md` for the release process and prerequisites. +## Benchmarks + +For new Harbor benchmark runs, use the standard workflow from the repository +root: `uv run --project benchmarks/harbor python -m harbor_adapter run --config +`. See `benchmarks/harbor/README.md` for configuration, verified APT +cache import, and the `--install-only` dependency preflight option. + +- Do not create per-job adapter copies or ad hoc bootstrap scripts. Add reusable + fixes and reviewed task-revision profiles to `benchmarks/harbor` with tests. +- Preserve task images, package sources, and official test scripts. Cache hits + must match the current package index's filename, size, and SHA256 digest. +- Run dependency preflight before model work when a task has a registered + profile. Record unsupported profiles explicitly; do not claim they passed. +- Keep setup failures and verifier errors separate from scored failures. Only + verifier timeouts may retry, at most three total attempts using the same agent + output and the task's original per-attempt time limit. + + ## Language Write everything in **English** by default — source code (identifiers, diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index fdd41cd..3b841aa 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -18,8 +18,81 @@ export ANTHROPIC_API_KEY="..." export ANTHROPIC_BASE_URL="https://gateway.example" ``` -From the repository root, run Harbor through this workspace and pin the agent -installed in the task container: +Run new benchmarks through the standard entry point, from the repository root: + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter run \ + --config benchmarks/harbor/configs/qemu-alpine-ssh.json +``` + +This example pins the QEMU task revision and uses `deepseek/deepseek-flash`. +Set the endpoint and credentials for that model, or copy the configuration and +change `agents[].model_name`, task selection, and budgets for another run. These +are ordinary Harbor JSON configurations; the entry point supplies the standard +verifier and cache policy. It accepts the repository's `NanoPyCodeAgent` adapter +with budget kwargs; credentials belong in the environment. + +The entry point builds the current working tree, including uncommitted changes, +into a wheel and checks its Python sources against the recorded snapshot. It +also snapshots the adapter, records source and wheel hashes, and uses that +snapshot throughout the run. Inputs and logs live in `jobs/-input/`; +Harbor outputs remain in `jobs//`. Existing jobs are never overwritten. +`summary.json` records scores, setup/preflight status, cache use, and verifier +attempts. An infrastructure exception has no score; it is not converted to zero. +The command exits nonzero for execution errors or incomplete results, while a +completed score of zero is a valid benchmark result. + +Use `--job-name ` for a readable run name. Add `--prepare-only` to +build and record inputs without starting containers. Add `--install-only` to +exercise dependency installation and preflight without calling the model or +executing the official tests. A full run still performs setup and preflight in +its own fresh container; an install-only check does not resume into model work. + +### Verified dependencies and preflight + +The standard adapter automatically uses `.cache/nanopy-harbor/`, shared across +runs and ignored by Git. Override it with `NANOPY_HARBOR_CACHE` or the entry +point's `--cache-dir`. On APT systems it refreshes the package index, resolves a +SHA256 download plan, restores matching cached files, and downloads missing +ones. Successful downloads are verified and saved before installation, so +APT cleanup hooks cannot erase the shared copies. A different version or digest +does not reuse an old cached file. Corrupt files fail setup before model work. +Other package managers use Harbor's regular dependency installation. + +To import previously verified downloads once: + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter import-apt \ + /path/to/bootstrap-cache/manifest.json +``` + +The manifest is a JSON array of objects with `url`, `filename`, `size`, and +`sha256`, accompanied by `packages/` files. Import validates every +file. At use time, the current APT index must independently match each entry. +Copying the entire shared cache to another machine is also supported. A new +machine with no matching cache still needs the original downloads to be +available; this mechanism does not replace package sources or invent missing +versions. + +Verifier dependency profiles are tied to reviewed task revisions in +`src/harbor_adapter/bootstrap.py`. The current QEMU profile installs `sshpass` +and warms the original verifier's uv 0.9.5, Python 3.13, pytest 8.4.1, and +pytest-json-ctrf 0.3.5 dependencies using `pytest --version`. It runs no task +assertions or solution code. Unknown QEMU revisions fail setup pending profile +review; tasks without a profile explicitly report `not_configured`. Add a +profile and regression tests when another task needs verifier preflight. + +Every installation writes `agent/bootstrap.json`, including partial progress +on failure. Preflight failure stops the trial before any model call. The +official verifier script remains unchanged and may still access the network +when it runs; a successful preflight is not a guarantee against later outages. +Full standard runs enable the timeout retry policy below by default, with zero +whole-trial retries. CI checks these contracts without model calls or Docker. + +### Direct Harbor invocation + +For adapter development or comparison with published versions, the lower-level +Harbor interface remains available. Pin the agent installed in the container: ```bash uv run --project benchmarks/harbor harbor run \ @@ -27,6 +100,8 @@ uv run --project benchmarks/harbor harbor run \ --agent harbor_adapter:NanoPyCodeAgent \ --agent-kwarg git_ref= \ --model anthropic/claude-sonnet-4-6 \ + --verifier harbor_adapter:RetryingVerifier \ + --verifier-timeout-multiplier 4 \ --env docker \ --n-concurrent 1 \ --n-attempts 1 @@ -65,6 +140,11 @@ The adapter also asks the agent to write an ATIF-v1.7 trajectory directly to ATIF output and backfills prompt, completion, cache-token, and cost totals into the agent result. Missing, invalid, or partial trajectories remain explicitly diagnosed; unknown usage or cost is not reported as zero. +If any costs are estimated, `final_metrics.extra` records +`cost_is_estimated: true` and the estimated portion in `estimated_cost_usd`. +The adapter preserves both fields in the agent result's `metadata.trajectory`, +including for partial totals. Estimates with a generation ID remain eligible +for billing reconciliation; a successful lookup replaces the estimate. By default, the adapter strips the first provider prefix from `--model` and passes the result to nanoPyCodeAgent as `ANTHROPIC_MODEL`. Set @@ -74,6 +154,39 @@ precedence. Harbor-native provider credentials and configured base URLs are also normalized to the `ANTHROPIC_*` variables expected by nanoPyCodeAgent's SDK. +## Retry verifier timeouts + +The example above uses `harbor_adapter:RetryingVerifier` for single-step Linux +tasks. It runs the original test script up to **three times total**, retrying +only when the script exceeds the task's native `[verifier].timeout_sec`. A +900-second task therefore gets at most three 900-second attempts. A completed +verification stops immediately, including a reward of zero. Script errors, +missing or invalid rewards, and cancellation do not trigger another attempt. + +`--verifier-timeout-multiplier 4` gives Harbor's outer watchdog room for all +three attempts plus test upload, process cleanup, and log collection. It does +not extend each test-script attempt. Conflicting timeout overrides or caps +that leave insufficient overall time are rejected. Use +`--verifier-kwarg max_attempts=2` for two total attempts, or `max_attempts=1` for +one; values above three are rejected. + +The runner reuses the same container, agent output, and dependency caches; +the agent is run once. Test scripts must tolerate being rerun in that +environment; verifier-side filesystem changes are not rolled back. Timed-out +process groups and other members of their Linux session are killed before +retrying. Each attempt's logs and any reward files are archived under +`verifier/attempts/1/`, `2/`, and `3/`. The runner clears current verifier +outputs before the next attempt, so a stale reward cannot determine its score. +`verifier/retry-summary.json` records each attempt's status, duration, and exit +code when available. Exhausting the attempts still produces Harbor's +`VerifierTimeoutError`. + +This verifier requires `python3` and `bash` in the container, which the +nanoPyCodeAgent adapter installs. These options apply to new runs; they cannot +restore a deleted container or missing task outputs from an old trial. +Harbor's ordinary `--max-retries` setting reruns the entire trial, including +the agent, and is independent of these verifier attempts. + ## Test the adapter ```bash diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index 3b84682..12ca806 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -17,7 +17,63 @@ export ANTHROPIC_API_KEY="..." export ANTHROPIC_BASE_URL="https://gateway.example" ``` -从仓库根目录通过这个 workspace 运行 Harbor，并固定安装到 task 容器中的 agent： +新的 benchmark 统一从仓库根目录通过标准入口运行： + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter run \ + --config benchmarks/harbor/configs/qemu-alpine-ssh.json +``` + +此示例固定了 QEMU 题目版本，模型为 `deepseek/deepseek-flash`。请配置对应的 +endpoint 和凭证；测试其他模型或题目时，复制配置并修改 `agents[].model_name`、 +题目选择及预算。这是普通的 Harbor JSON 配置，入口会补齐标准验证器和缓存策略。 +入口接受本仓库的 `NanoPyCodeAgent` adapter 和预算参数，凭证通过环境变量传入。 + +入口会将当前工作区（包括未提交修改）构建为 wheel，并核对其 Python 源码与保存的 +快照一致；同时保存 adapter 快照及源码、wheel 的哈希，整个运行都使用该快照。 +输入和日志保存在 `jobs/-input/`，Harbor 输出保存在 +`jobs//`，不会覆盖已有运行。`summary.json` 记录评分、安装和预检状态、 +缓存使用情况及验证次数。基础设施异常保留为无评分，不会转换为 0 分。 +执行错误或结果不完整时命令返回非零退出码；正常完成验证得到 0 分仍是有效结果。 + +可用 `--job-name ` 命名运行。`--prepare-only` 只构建并保存输入， +不启动容器；`--install-only` 只执行依赖安装和预检，不调用模型、不运行官方测试。 +完整运行仍会在自己的新容器里执行安装和预检，不能把 install-only 容器续跑为解题。 + +### 依赖缓存与预检 + +标准 adapter 自动使用被 Git 忽略的 `.cache/nanopy-harbor/` 跨运行共享缓存， +可通过 `NANOPY_HARBOR_CACHE` 或入口的 `--cache-dir` 覆盖。对于 APT 系统，先更新 +索引并解析 SHA256 下载计划，再恢复匹配的缓存文件、下载缺失文件。下载成功后会先 +校验并保存缓存，再安装，避免 APT 清理钩子删除共享副本。版本或哈希不匹配时不会 +复用旧包，文件损坏会在模型调用前终止安装。其他包管理器使用 Harbor 的常规安装。 + +历史下载只需导入一次： + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter import-apt \ + /path/to/bootstrap-cache/manifest.json +``` + +manifest 是包含 `url`、`filename`、`size`、`sha256` 的 JSON 对象数组，同目录下 +配套 `packages/` 文件。导入时逐个校验，使用时还必须独立匹配当前 APT +索引。也可将整个共享缓存复制到其他机器。新机器没有匹配缓存时仍依赖原下载地址 +可用；此机制不会更换软件源或伪造缺失版本。 + +验证器依赖预检配置位于 `src/harbor_adapter/bootstrap.py`，绑定已检查的题目版本。 +目前 QEMU 配置会安装 `sshpass`，并通过 `pytest --version` 预热原验证器所需的 +uv 0.9.5、Python 3.13、pytest 8.4.1 和 pytest-json-ctrf 0.3.5，不执行题目断言或 +参考解。未知的 QEMU 版本会在安装阶段报错，要求先检查对应预检配置；没有预检配置 +的其他题目明确记录为 `not_configured`。其他题目需要预检时，在这里新增配置及回归测试。 + +每次安装都会写入 `agent/bootstrap.json`，失败时也保留已完成的步骤。预检失败会 +在模型调用前终止 trial。官方验证脚本保持原样，执行时仍可能访问网络，所以预检 +通过不能保证后续没有网络故障。完整标准运行默认启用下述验证超时重试，整题重试 +次数为零。CI 会验证这些约定，不调用模型、不启动 Docker。 + +### 直接调用 Harbor + +开发 adapter 或对照已发布版本时，仍可使用底层 Harbor 入口，并固定容器内 agent 版本： ```bash uv run --project benchmarks/harbor harbor run \ @@ -25,6 +81,8 @@ uv run --project benchmarks/harbor harbor run \ --agent harbor_adapter:NanoPyCodeAgent \ --agent-kwarg git_ref= \ --model anthropic/claude-sonnet-4-6 \ + --verifier harbor_adapter:RetryingVerifier \ + --verifier-timeout-multiplier 4 \ --env docker \ --n-concurrent 1 \ --n-attempts 1 @@ -56,6 +114,10 @@ adapter 还会要求 agent 将 ATIF-v1.7 trajectory 直接写入 `/logs/agent/trajectory.json`。Harbor 会把该文件作为 trial 的原生 ATIF 输出采集， 并将 prompt、completion、cache token 和 cost 汇总回填到 agent result。trajectory 缺失、无效或指标不完整时会保留明确诊断；未知 usage 或 cost 不会被记成零。 +只要费用中包含估算，`final_metrics.extra` 就会记录 `cost_is_estimated: true`， +并通过 `estimated_cost_usd` 单独记录估算部分的金额。adapter 会将这两个字段保留到 +agent result 的 `metadata.trajectory`，费用仅部分已知时也一样。带 generation ID +的估算费用仍会进行账单补查；补查成功后，真实费用会替换估算值。 默认情况下，adapter 会移除 `--model` 中的第一个 provider 前缀，并将结果作为 `ANTHROPIC_MODEL` 传给 nanoPyCodeAgent。只有自定义 endpoint 要求的实际模型名与 @@ -63,6 +125,30 @@ Harbor 的 `provider/model` 身份不同时，才显式设置 `ANTHROPIC_MODEL` Harbor 原生的 provider 凭证和已经配置的 base URL 也会转换为 nanoPyCodeAgent SDK 所需的 `ANTHROPIC_*` 变量。 +## 验证器超时重试 + +上面的示例使用 `harbor_adapter:RetryingVerifier`，适用于单阶段 Linux 题目。 +原始测试脚本**总共最多执行 3 次**，仅当脚本超过题目原生的 +`[verifier].timeout_sec` 时才重试。900 秒时限的题目，每次仍最多执行 900 秒。 +验证完成后立即停止，包括得分为 0 的情况。脚本错误、评分缺失或格式错误、取消执行 +都不会触发下一次尝试。 + +`--verifier-timeout-multiplier 4` 为 Harbor 外层时限预留 3 次尝试以及上传测试、 +清理进程和收集日志的时间，不会延长每次测试脚本的时限。如果其他超时覆盖或上限 +导致总时间不足，会报错。可用 `--verifier-kwarg max_attempts=2` 将总尝试次数设为 +2 次，或用 `max_attempts=1` 只执行一次；不允许设置为超过 3 次。 + +重试复用同一容器、agent 产物和依赖缓存，agent 只运行一次。测试脚本需要能够在该 +环境中重复执行，验证过程对文件系统的修改不会回滚。超时后会先终止测试进程组以及 +同一 Linux session 中的其他进程，再开始下一次尝试。每次日志及评分文件归档到 +`verifier/attempts/1/`、`2/`、`3/`。下一次尝试开始前会清理当前验证输出，避免旧 +评分被误用。`verifier/retry-summary.json` 记录每次状态、耗时及可用的退出码。 +所有尝试均超时后，仍以 Harbor 的 `VerifierTimeoutError` 结束。 + +容器需要 `python3` 和 `bash`，nanoPyCodeAgent adapter 会安装它们。这些选项用于 +新运行，无法恢复旧 trial 已删除的容器或缺失的任务产物。Harbor 原生的 +`--max-retries` 会重新执行整道题（包括 agent），与这里的验证尝试次数相互独立。 + ## 测试 adapter ```bash diff --git a/benchmarks/harbor/configs/qemu-alpine-ssh.json b/benchmarks/harbor/configs/qemu-alpine-ssh.json new file mode 100644 index 0000000..e33e613 --- /dev/null +++ b/benchmarks/harbor/configs/qemu-alpine-ssh.json @@ -0,0 +1,21 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/qemu-alpine-ssh", + "ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28" + } + ] +} diff --git a/benchmarks/harbor/reports/deepseek-official-20260930.md b/benchmarks/harbor/reports/deepseek-official-20260930.md new file mode 100644 index 0000000..4b2af9c --- /dev/null +++ b/benchmarks/harbor/reports/deepseek-official-20260930.md @@ -0,0 +1,86 @@ +# DeepSeek official endpoint pilot20 + +Switched the 20-task pilot set from OpenRouter to DeepSeek's official +Anthropic-compatible endpoint and re-ran it with `deepseek-flash`. The run +scored **16/18 scored trials** (Harbor mean 0.80); two trials were lost to +infrastructure failures rather than model failures. + +The complete configuration, per-trial usage, token-estimated cost, and balance +snapshots are in the +[public results](../results/tb21-deepseek-official-pilot20-20260930.json). Raw +evidence remains in the recorded Git-ignored `jobs/` directories. + +## Configuration + +| Setting | Value | +| --- | --- | +| Job | `tb21-deepseek-official-pilot20-20260930` | +| Agent | `e75cacb6` (`0.8.1.dev45`), locally built and verified wheel | +| Endpoint | `https://api.deepseek.com/anthropic` (Anthropic Messages) | +| Model | `deepseek-flash` | +| Tasks | the pinned 20-task pilot set | +| Budget | max_turns 100, max_tokens 65536, per-task agent budget native−180 s | +| Concurrency / attempts / retries | 2 / 1 / 0 | +| Provider pin / HTTP recorder | none — the provider is the only change | + +## Result + +| Bucket | Count | +| --- | ---: | +| Planned trials | 20 | +| Scored trials | 18 | +| Passed | 16 | +| Reward 0 | 2 | +| Infrastructure exceptions | 2 | +| Pass rate over scored | 88.9% | + +- Reward 0: `torch-tensor-parallelism`, `pytorch-model-recovery`. +- `qemu-alpine-ssh`: agent setup failed fetching pinned Debian packages with + HTTP 404; no model call happened. +- `torch-pipeline-parallelism`: the agent completed, but the official verifier + exceeded its native 900 s limit. + +Token usage: 30,933,108 prompt (30,586,240 cached, 346,868 uncached), 891,020 +completion. + +## Cost without a provider cost API + +DeepSeek's official API reports token usage but no per-request cost, no +generation id, and no usage/billing endpoint — `usage.cost` is absent, +`x-generation-id` is absent, and `/v1/generation`, `/user/usage`, +`/billing/usage`, and `/pricing` all return 404. The previous OpenRouter +reconciliation path therefore cannot resolve a DeepSeek cost. + +This branch adds a **token-based estimator** (see Implementation) that prices +the returned tokens with a static DeepSeek price list (USD per million: +input 0.3, cache read 0.006, cache write 0, output 1.2). It marks the result +`kind: estimated`, so it stays distinguishable from provider-reported cost. + +| Component | Tokens | Rate | Cost (USD) | +| --- | ---: | ---: | ---: | +| Uncached input | 346,868 | 0.3 | 0.104060 | +| Cache read | 30,586,240 | 0.006 | 0.183517 | +| Output | 891,020 | 1.2 | 1.069224 | +| **Total** | | | **1.356802** | + +An account balance delta (`/user/balance`, ¥106.63 → ¥102.01) is recorded as an +upper bound only: the interactive assistant session used the same account +during the run, so it is not an isolated run cost, and it is denominated in CNY. + +## Implementation + +- `cost.py`: add `estimated_cost(model, usage)` and a static DeepSeek price + table; `agent.py` uses it only after `usage_cost` fails, so an explicit + provider cost always wins. +- `harbor_adapter`: retry the system-dependency `apt-get` transaction once or + twice on a transient HTTP 404 before failing setup. In the follow-up qemu + re-run the retry fired on attempts 1/3 and 2/3, but the pinned + `bullseye-security` index references `.deb` versions that are gone, so the 404 + is deterministic and a retry alone cannot fix that task. The historical fix + preloads a verified `.deb` cache from the original APT index. + +## Comparison limits + +The OpenRouter pilot20 runs used provider pinning, a different snapshot date, +and different instrumentation, and this run has two infrastructure exceptions. +This is a provider-switch record, not a controlled causal comparison. diff --git a/benchmarks/harbor/results/tb21-deepseek-official-pilot20-20260930.json b/benchmarks/harbor/results/tb21-deepseek-official-pilot20-20260930.json new file mode 100644 index 0000000..ebd66c2 --- /dev/null +++ b/benchmarks/harbor/results/tb21-deepseek-official-pilot20-20260930.json @@ -0,0 +1,387 @@ +{ + "schema_version": 1, + "generated_at": "2026-10-01T12:28:08.216679+00:00", + "job_name": "tb21-deepseek-official-pilot20-20260930", + "run_date": "2026-09-30", + "scope": "Single-attempt provider switch of the 20-task pilot set from OpenRouter to the DeepSeek official Anthropic-compatible endpoint; not a full Terminal-Bench 2.1 score or a controlled causal comparison.", + "experiment": { + "agent_ref": "e75cacb6a780314471ff12655ecb6fe414109ee4", + "wheel_sha256": "2be853f2dcbaef286295aacfbc2c074455d41b8531d7d585fdc727eeecaa9d73", + "harbor_version": "0.21.0", + "dataset": "terminal-bench/terminal-bench-2-1", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "api": "Anthropic Messages", + "model_settings_source": "~/.nanoPyCodeAgent/settings.json", + "limits": { + "n_tasks": 20, + "n_attempts": 1, + "n_concurrent": 2, + "max_turns": 100, + "max_tokens_per_response": 65536, + "time_budget": "native task timeout - 180s, computed per task", + "harbor_max_retries": 0, + "harbor_timeout_overrides": false, + "openrouter_provider_pin": null, + "http_recorder": null, + "temperature": "not explicitly set", + "reasoning_effort": "not explicitly set" + } + }, + "scores": { + "planned_trials": 20, + "scored_trials": 18, + "passed_trials": 16, + "failed_reward_trials": 2, + "unscored_trials": 2, + "exception_counts": { + "VerifierTimeoutError": 1, + "NonZeroAgentExitCodeError": 1 + }, + "harbor_eval_mean": 0.8, + "pass_rate_over_planned": 0.8, + "pass_rate_over_scored": 0.8888888888888888 + }, + "usage": { + "total_prompt_tokens": 30933108, + "total_cached_tokens": 30586240, + "total_cache_write_tokens": 0, + "total_uncached_input_tokens": 346868, + "total_completion_tokens": 891020, + "complete": true + }, + "cost": { + "currency": "USD", + "method": "token_estimate.deepseek", + "source": "static DeepSeek official price table", + "prices_per_million_tokens": { + "input": "0.3", + "output": "1.2", + "cache_read": "0.006", + "cache_write": "0" + }, + "token_estimated_total_usd": "1.35680184", + "reported_by_provider": false, + "note": "DeepSeek's official API returns token usage but no per-request cost, no generation id, and no usage/billing endpoint. This total is computed from token usage, not reported by the provider.", + "account_balance_snapshots_cny": [ + { + "label": "before", + "at": "2026-09-30T14:05:18.075037+00:00", + "is_available": true, + "balance_infos": [ + { + "currency": "CNY", + "total_balance": "106.63", + "granted_balance": "0.00", + "topped_up_balance": "106.63" + } + ] + }, + { + "label": "after", + "at": "2026-09-30T16:14:44.312896+00:00", + "is_available": true, + "balance_infos": [ + { + "currency": "CNY", + "total_balance": "102.01", + "granted_balance": "0.00", + "topped_up_balance": "102.01" + } + ] + } + ], + "account_balance_delta_cny": "4.62", + "account_balance_caveat": "The interactive assistant session used the same DeepSeek account during the run, so the balance delta is an upper bound, not an isolated run cost." + }, + "trials": [ + { + "task": "terminal-bench/caffe-cifar-10", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 3600, + "usage": { + "prompt_tokens": 932801, + "cache_read_tokens": 914048, + "cache_write_tokens": 0, + "uncached_input_tokens": 18753, + "completion_tokens": 25539 + }, + "estimated_cost_usd": "0.041756988" + }, + { + "task": "terminal-bench/circuit-fibsqrt", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 3600, + "usage": { + "prompt_tokens": 972406, + "cache_read_tokens": 963328, + "cache_write_tokens": 0, + "uncached_input_tokens": 9078, + "completion_tokens": 64174 + }, + "estimated_cost_usd": "0.085512168" + }, + { + "task": "terminal-bench/dna-assembly", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 1800, + "usage": { + "prompt_tokens": 2356256, + "cache_read_tokens": 2331008, + "cache_write_tokens": 0, + "uncached_input_tokens": 25248, + "completion_tokens": 88429 + }, + "estimated_cost_usd": "0.127675248" + }, + { + "task": "terminal-bench/kv-store-grpc", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 32653, + "cache_read_tokens": 29568, + "cache_write_tokens": 0, + "uncached_input_tokens": 3085, + "completion_tokens": 2825 + }, + "estimated_cost_usd": "0.004492908" + }, + { + "task": "terminal-bench/llm-inference-batching-scheduler", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 1800, + "usage": { + "prompt_tokens": 1079376, + "cache_read_tokens": 1061248, + "cache_write_tokens": 0, + "uncached_input_tokens": 18128, + "completion_tokens": 52222 + }, + "estimated_cost_usd": "0.074472288" + }, + { + "task": "terminal-bench/log-summary-date-ranges", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 27223, + "cache_read_tokens": 22656, + "cache_write_tokens": 0, + "uncached_input_tokens": 4567, + "completion_tokens": 1777 + }, + "estimated_cost_usd": "0.003638436" + }, + { + "task": "terminal-bench/merge-diff-arc-agi-task", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 274857, + "cache_read_tokens": 260224, + "cache_write_tokens": 0, + "uncached_input_tokens": 14633, + "completion_tokens": 16171 + }, + "estimated_cost_usd": "0.025356444" + }, + { + "task": "terminal-bench/model-extraction-relu-logits", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 538556, + "cache_read_tokens": 526592, + "cache_write_tokens": 0, + "uncached_input_tokens": 11964, + "completion_tokens": 29094 + }, + "estimated_cost_usd": "0.041661552" + }, + { + "task": "terminal-bench/mteb-leaderboard", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 3600, + "usage": { + "prompt_tokens": 1719176, + "cache_read_tokens": 1680128, + "cache_write_tokens": 0, + "uncached_input_tokens": 39048, + "completion_tokens": 29412 + }, + "estimated_cost_usd": "0.057089568" + }, + { + "task": "terminal-bench/openssl-selfsigned-cert", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 40143, + "cache_read_tokens": 37120, + "cache_write_tokens": 0, + "uncached_input_tokens": 3023, + "completion_tokens": 3902 + }, + "estimated_cost_usd": "0.00581202" + }, + { + "task": "terminal-bench/path-tracing", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 1800, + "usage": { + "prompt_tokens": 1140749, + "cache_read_tokens": 1084672, + "cache_write_tokens": 0, + "uncached_input_tokens": 56077, + "completion_tokens": 35038 + }, + "estimated_cost_usd": "0.065376732" + }, + { + "task": "terminal-bench/pypi-server", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 110863, + "cache_read_tokens": 105216, + "cache_write_tokens": 0, + "uncached_input_tokens": 5647, + "completion_tokens": 5713 + }, + "estimated_cost_usd": "0.009180996" + }, + { + "task": "terminal-bench/pytorch-model-recovery", + "reward": 0.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 1481698, + "cache_read_tokens": 1460352, + "cache_write_tokens": 0, + "uncached_input_tokens": 21346, + "completion_tokens": 44263 + }, + "estimated_cost_usd": "0.068281512" + }, + { + "task": "terminal-bench/qemu-alpine-ssh", + "reward": null, + "exception_type": "NonZeroAgentExitCodeError", + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "uncached_input_tokens": 0, + "completion_tokens": 0 + }, + "estimated_cost_usd": "0.000" + }, + { + "task": "terminal-bench/regex-chess", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 3600, + "usage": { + "prompt_tokens": 6867208, + "cache_read_tokens": 6828288, + "cache_write_tokens": 0, + "uncached_input_tokens": 38920, + "completion_tokens": 122473 + }, + "estimated_cost_usd": "0.199613328" + }, + { + "task": "terminal-bench/regex-log", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 435322, + "cache_read_tokens": 429824, + "cache_write_tokens": 0, + "uncached_input_tokens": 5498, + "completion_tokens": 62276 + }, + "estimated_cost_usd": "0.078959544" + }, + { + "task": "terminal-bench/schemelike-metacircular-eval", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 2400, + "usage": { + "prompt_tokens": 11204578, + "cache_read_tokens": 11154688, + "cache_write_tokens": 0, + "uncached_input_tokens": 49890, + "completion_tokens": 192178 + }, + "estimated_cost_usd": "0.312508728" + }, + { + "task": "terminal-bench/torch-pipeline-parallelism", + "reward": null, + "exception_type": "VerifierTimeoutError", + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 1334231, + "cache_read_tokens": 1318656, + "cache_write_tokens": 0, + "uncached_input_tokens": 15575, + "completion_tokens": 62393 + }, + "estimated_cost_usd": "0.087456036" + }, + { + "task": "terminal-bench/torch-tensor-parallelism", + "reward": 0.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 168404, + "cache_read_tokens": 164992, + "cache_write_tokens": 0, + "uncached_input_tokens": 3412, + "completion_tokens": 18918 + }, + "estimated_cost_usd": "0.024715152" + }, + { + "task": "terminal-bench/write-compressor", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 216608, + "cache_read_tokens": 213632, + "cache_write_tokens": 0, + "uncached_input_tokens": 2976, + "completion_tokens": 34223 + }, + "estimated_cost_usd": "0.043242192" + } + ], + "qemu_alpine_ssh_retry": { + "job": "tb21-deepseek-official-qemu-retry-20260930", + "exception_type": "NonZeroAgentExitCodeError", + "retry": "The tracked 404 retry fired on attempts 1/3 and 2/3 and still failed.", + "finding": "The pinned Debian bullseye-security index references .deb versions that are no longer in the pool; the 404 is deterministic, so a retry alone cannot fix this task. The historical fix preloads a verified .deb cache from the original APT index." + }, + "comparison_limits": "The OpenRouter pilot20 runs used provider pinning, a different snapshot date, and different instrumentation. Two exceptions here are infrastructure failures (apt 404 and a verifier timeout), not model failures." +} diff --git a/benchmarks/harbor/src/harbor_adapter/__init__.py b/benchmarks/harbor/src/harbor_adapter/__init__.py index 915d6fd..5e8c83c 100644 --- a/benchmarks/harbor/src/harbor_adapter/__init__.py +++ b/benchmarks/harbor/src/harbor_adapter/__init__.py @@ -1,5 +1,6 @@ """Public import for the repository-local nanoPyCodeAgent Harbor adapter.""" from .adapter import NanoPyCodeAgent +from .verifier import RetryingVerifier -__all__ = ["NanoPyCodeAgent"] +__all__ = ["NanoPyCodeAgent", "RetryingVerifier"] diff --git a/benchmarks/harbor/src/harbor_adapter/__main__.py b/benchmarks/harbor/src/harbor_adapter/__main__.py new file mode 100644 index 0000000..d41cf39 --- /dev/null +++ b/benchmarks/harbor/src/harbor_adapter/__main__.py @@ -0,0 +1,182 @@ +"""Repository benchmark entry point with recorded source and bootstrap policy.""" + +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +from zipfile import ZipFile + +from harbor.models.job.config import JobConfig + +from .bootstrap import AptCache, default_cache_dir + + +def standard_config(raw: dict) -> JobConfig: + """Validate policy before building or creating a benchmark environment.""" + raw = json.loads(json.dumps(raw)) + raw.setdefault("retry", {"max_retries": 0}) + raw.setdefault("n_concurrent_trials", 1) + raw.setdefault("verifier", { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": {"max_attempts": 3}, + }) + raw.setdefault("verifier_timeout_multiplier", 4) + config = JobConfig.model_validate(raw) + if config.retry.max_retries != 0: + raise ValueError("The standard workflow disables whole-trial retries") + if config.verifier.import_path != "harbor_adapter:RetryingVerifier": + raise ValueError("The standard workflow requires RetryingVerifier") + attempts = config.verifier.kwargs.get("max_attempts", 3) + if type(attempts) is not int or attempts != 3: + raise ValueError("The standard workflow requires max_attempts=3") + if ( + config.verifier_timeout_multiplier != 4 + or config.verifier.override_timeout_sec is not None + or config.verifier.max_timeout_sec is not None + ): + raise ValueError( + "Use native per-attempt timeouts and verifier_timeout_multiplier=4" + ) + if (config.verifier.disable and not config.install_only) or config.source_jobs: + raise ValueError("The standard workflow runs fresh trials with verification enabled") + if not config.tasks and not config.datasets: + raise ValueError("Select at least one task or dataset") + if not config.agents or config.n_attempts < 1: + raise ValueError("Select at least one agent and one trial attempt") + for agent in config.agents: + identifiers = [value for value in (agent.import_path, agent.name) if value] + if not identifiers or any( + value != "harbor_adapter:NanoPyCodeAgent" for value in identifiers + ): + raise ValueError("The standard workflow requires harbor_adapter:NanoPyCodeAgent") + if agent.env or set(agent.kwargs) - { + "max_turns", "max_tokens", "time_budget_seconds", + }: + raise ValueError( + "Pass model credentials through the environment; " + "agent kwargs may only set budgets" + ) + if not agent.model_name: + raise ValueError("Each agent requires an explicit model_name") + return config + + +def summarize(job_dir: Path, *, install_only: bool = False) -> dict: + trials = [] + for path in sorted(job_dir.glob("*/result.json")): + result = json.loads(path.read_text()) + bootstrap_path = path.parent / "agent/bootstrap.json" + bootstrap = json.loads(bootstrap_path.read_text()) if bootstrap_path.exists() else {} + exception = result.get("exception_info") + verifier = result.get("verifier_result") or {} + retry_path = path.parent / "verifier/retry-summary.json" + if exception: + status = "setup_failed" if bootstrap.get("status") == "failed" else "execution_error" + elif install_only and bootstrap.get("status") == "ready": + status = "setup_passed" + else: + status = "scored" if verifier.get("rewards") is not None else "unscored" + trials.append({ + "task": result.get("task_name"), "trial": path.parent.name, + "status": status, + "exception_type": exception.get("exception_type") if exception else None, + "rewards": None if exception else verifier.get("rewards"), + "bootstrap": bootstrap, + "verifier_attempts": json.loads(retry_path.read_text()) if retry_path.exists() else None, + }) + return {"trials": trials} + + +def run(args) -> int: + root = Path.cwd() + if not (root / "src/nanopycodeagent").is_dir(): + raise ValueError("Run from the nanoPyCodeAgent repository root") + config = standard_config(json.loads(args.config.read_text())) + if args.job_name: + config.job_name = args.job_name + if Path(config.job_name).name != config.job_name or config.job_name in {".", ".."}: + raise ValueError("job_name must be a single directory name") + config.install_only = args.install_only or config.install_only + config.jobs_dir = config.jobs_dir.resolve() + job_dir = config.jobs_dir / config.job_name + record = config.jobs_dir / (config.job_name + "-input") + if job_dir.exists() or record.exists(): + raise ValueError("Use a new job_name; existing jobs and inputs are never overwritten") + record.mkdir(parents=True) + record.chmod(0o700) + source = record / "source" + shutil.copytree(root / "benchmarks/harbor/src/harbor_adapter", source / "harbor_adapter", ignore=shutil.ignore_patterns("__pycache__")) + shutil.copytree(root / "src/nanopycodeagent", source / "nanopycodeagent", ignore=shutil.ignore_patterns("__pycache__")) + subprocess.run(["uv", "build", "--wheel", "--out-dir", str(record / "dist")], check=True) + wheel, = (record / "dist").glob("*.whl") + with ZipFile(wheel) as archive: + for path in (source / "nanopycodeagent").rglob("*.py"): + if archive.read(path.relative_to(source).as_posix()) != path.read_bytes(): + raise ValueError("Source changed during preparation; prepare a new job") + cache = args.cache_dir.expanduser().resolve() + for agent in config.agents: + agent.kwargs.update(wheel_path=str(wheel), bootstrap_cache=str(cache)) + config_path = record / "config.json" + config_path.write_text(config.model_dump_json(indent=2) + "\n") + manifest = { + "created_at": datetime.now(timezone.utc).isoformat(), + "commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + "branch": subprocess.check_output(["git", "branch", "--show-current"], text=True).strip(), + "source_kind": "working tree including uncommitted files", + "source_sha256": {str(p.relative_to(source)): hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(source.rglob("*.py"))}, + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), + "cache_dir": str(cache), + } + (record / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + (record / "working-tree.patch").write_bytes(subprocess.check_output(["git", "diff", "HEAD"])) + print(f"Prepared {record}", flush=True) + if args.prepare_only: + return 0 + env = dict(os.environ) + env["PYTHONPATH"] = str(source) + env["PATH"] = str(Path(sys.executable).parent) + os.pathsep + env["PATH"] + code = None + try: + with (record / "harbor.log").open("w") as log: + code = subprocess.run( + ["harbor", "run", "--config", str(config_path), "--yes"], + env=env, stdout=log, stderr=subprocess.STDOUT, + ).returncode + finally: + summary = summarize(job_dir, install_only=config.install_only) + summary["harbor_exit_code"] = code + (record / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(f"Results: {record / 'summary.json'}", flush=True) + incomplete = not summary["trials"] or any( + row["status"] not in {"scored", "setup_passed"} for row in summary["trials"] + ) + return code or int(incomplete) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + benchmark = commands.add_parser("run", help="Record the working tree and run the standard Harbor workflow") + benchmark.add_argument("--config", type=Path, required=True) + benchmark.add_argument("--job-name") + benchmark.add_argument("--cache-dir", type=Path, default=default_cache_dir()) + benchmark.add_argument("--prepare-only", action="store_true", help="Build and record inputs without starting containers or calling the model") + benchmark.add_argument("--install-only", action="store_true", help="Run setup and dependency preflight without calling the model or verifier") + cache = commands.add_parser("import-apt", help="Import a manifest and its sibling packages/ directory") + cache.add_argument("manifest", type=Path) + cache.add_argument("--cache-dir", type=Path, default=default_cache_dir()) + args = parser.parse_args() + if args.command == "import-apt": + count = AptCache(args.cache_dir.expanduser().resolve()).import_manifest(args.manifest) + print(f"Imported {count} verified APT packages into {args.cache_dir}") + return 0 + return run(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py b/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py new file mode 100644 index 0000000..da4f84c --- /dev/null +++ b/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py @@ -0,0 +1,143 @@ +"""Run a Linux verifier inside its container with bounded timeout retries. + +This file is sent to the container as Python source and uses only the standard +library. It never starts an agent or changes the task's test script. +""" + +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess +import sys +import time + + +def _stop_process(process): + # GNU timeout and other tools may create additional process groups in the + # test's session. Snapshot them before killing the session leader. + members = [] + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + fields = (entry / "stat").read_text().rsplit(")", 1)[1].split() + if int(fields[3]) == process.pid: + members.append(int(entry.name)) + except (OSError, ValueError, IndexError): + continue + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + for pid in members: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) + + +def _archive(logs, attempt): + target = logs / "attempts" / str(attempt) + target.mkdir(parents=True) + for source in logs.iterdir(): + if source.name in {"attempts", "retry-summary.json"}: + continue + if source.is_dir(): + shutil.copytree(source, target / source.name) + else: + shutil.copy2(source, target / source.name) + + +def run(command, timeout_sec, max_attempts, logs, control): + """Return execution status; only our own elapsed deadline is retryable.""" + logs = Path(logs) + control = Path(control) + cancelled = control.with_suffix(".cancelled") + attempts = [] + summary = {"timeout_sec": timeout_sec, "max_attempts": max_attempts, + "attempts": attempts} + + def interrupted(signum, frame): + raise InterruptedError("Verifier execution cancelled") + + signal.signal(signal.SIGTERM, interrupted) + signal.signal(signal.SIGINT, interrupted) + control.write_text(str(os.getpid())) + try: + for attempt in range(1, max_attempts + 1): + if cancelled.exists(): + raise InterruptedError("Verifier execution cancelled") + # A timed-out test may have written a reward before hanging. It + # must never be mistaken for the next attempt's result. + for previous in logs.iterdir(): + if previous.name in {"attempts", "retry-summary.json"}: + continue + if previous.is_dir(): + shutil.rmtree(previous) + else: + previous.unlink() + started = time.monotonic() + record = {"attempt": attempt, "status": "error"} + attempts.append(record) + process = None + try: + process = subprocess.Popen( + ["bash", "-c", command], stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + start_new_session=True, + ) + try: + record["return_code"] = process.wait(timeout=timeout_sec) + record["status"] = "completed" + except subprocess.TimeoutExpired: + _stop_process(process) + record["status"] = "timeout" + except BaseException: + if process is not None: + _stop_process(process) + record["status"] = "cancelled" + raise + finally: + record["duration_seconds"] = time.monotonic() - started + _archive(logs, attempt) + # Remove even the last timed-out attempt's reward from the + # canonical location; the original stays in its archive. + if record["status"] != "completed": + for name in ("reward.txt", "reward.json"): + (logs / name).unlink(missing_ok=True) + (logs / "retry-summary.json").write_text(json.dumps(summary, indent=2)) + if record["status"] == "completed": + return record + return attempts[-1] + finally: + control.unlink(missing_ok=True) + + +def cancel(control): + """Stop this runner, or prevent it starting after a transport cancellation.""" + control = Path(control) + control.with_suffix(".cancelled").touch() + try: + pid = int(control.read_text()) + except FileNotFoundError: + return + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + return + for _ in range(100): + if not control.exists(): + return + time.sleep(0.05) + raise RuntimeError("Verifier runner did not stop after cancellation") + + +if __name__ == "__main__": + if sys.argv[1] == "cancel": + cancel(sys.argv[2]) + else: + command, timeout, count, logs, control = sys.argv[1:] + print(json.dumps(run(command, float(timeout), int(count), logs, control))) diff --git a/benchmarks/harbor/src/harbor_adapter/adapter.py b/benchmarks/harbor/src/harbor_adapter/adapter.py index 621eebb..8d92253 100644 --- a/benchmarks/harbor/src/harbor_adapter/adapter.py +++ b/benchmarks/harbor/src/harbor_adapter/adapter.py @@ -1,5 +1,9 @@ """Harbor adapter for running nanoPyCodeAgent in benchmark containers.""" +import asyncio +import hashlib +import json +from pathlib import Path import re import shlex import uuid @@ -8,6 +12,7 @@ from harbor.agents.installed.base import ( BaseInstalledAgent, CliFlag, + NonZeroAgentExitCodeError, with_prompt_template, ) from harbor.agents.model_connection import ModelConnectionSpec @@ -15,7 +20,13 @@ from harbor.models.agent.context import AgentContext from harbor.models.trajectories.trajectory import Trajectory +from .bootstrap import AptCache, default_cache_dir, verifier_profile + _DEFAULT_MAX_TURNS = 50 +_SYSTEM_DEPENDENCIES = ("curl", "bash", "git", "python3", "ca_certificates") +# A 404 can reflect a stale index or an unavailable file. Restore verified +# downloads first, then retry the package transaction at most twice. +_DEPENDENCY_RETRY_DELAYS = (2.0, 4.0) _PACKAGE_NAME = "nanoPyCodeAgent" _REPOSITORY_URL = "https://github.com/minixalpha/nanoPyCodeAgent.git" _UV_VERSION = "0.9.11" @@ -42,7 +53,10 @@ class NanoPyCodeAgent(BaseInstalledAgent): CliFlag("time_budget_seconds", cli="--time-budget-seconds", type="int"), ] - def __init__(self, *args, git_ref: str | None = None, **kwargs): + def __init__( + self, *args, git_ref: str | None = None, wheel_path: str | None = None, + bootstrap_cache: str | None = None, **kwargs, + ): if git_ref is not None: if not isinstance(git_ref, str): raise ValueError( @@ -52,9 +66,22 @@ def __init__(self, *args, git_ref: str | None = None, **kwargs): git_ref = git_ref.strip() if not git_ref: raise ValueError("git_ref must not be blank") - if git_ref is not None and kwargs.get("version") is not None: - raise ValueError("version and git_ref are mutually exclusive") + pins = (git_ref, wheel_path, kwargs.get("version")) + if sum(pin is not None for pin in pins) > 1: + raise ValueError("version, git_ref, and wheel_path are mutually exclusive") self._git_ref = git_ref + self._wheel_path = ( + Path(wheel_path).expanduser().resolve() if wheel_path else None + ) + if self._wheel_path is not None and ( + not self._wheel_path.is_file() or self._wheel_path.suffix != ".whl" + ): + raise ValueError("wheel_path must point to an existing wheel") + self._bootstrap_cache = AptCache( + Path(bootstrap_cache).expanduser().resolve() + if bootstrap_cache else default_cache_dir() + ) + self._bootstrap_record: dict = {} super().__init__(*args, **kwargs) max_tokens = self._resolved_flags.get("max_tokens") if max_tokens is not None and max_tokens < 1: @@ -78,6 +105,8 @@ def parse_version(self, stdout: str) -> str: return match.group(1) if match else lines[-1] def _install_target(self) -> str: + if self._wheel_path is not None: + return "/tmp/" + self._wheel_path.name if self._git_ref is not None: return f"git+{_REPOSITORY_URL}@{self._git_ref}" if self._version is not None: @@ -86,10 +115,49 @@ def _install_target(self) -> str: @override async def install(self, environment: BaseEnvironment) -> None: - await self.ensure_system_dependencies( - environment, - ("curl", "bash", "git", "python3", "ca_certificates"), + record = self._bootstrap_record + record.update( + status="running", stage="profile", + cache_dir=str(self._bootstrap_cache.root), ) + try: + profile = verifier_profile( + self.logs_dir, getattr(environment, "environment_name", None), + ) + record["verifier_preflight"] = { + "profile": profile.name if profile else None, + "status": "pending" if profile else "not_configured", + } + record["stage"] = "system_dependencies" + await self._ensure_system_dependencies(environment) + record["stage"] = "agent_install" + await self._install_agent(environment) + if profile: + record["stage"] = "verifier_dependencies" + await self._install_apt(environment, profile.apt_packages) + await self.exec_as_agent(environment, command=profile.command) + record["verifier_preflight"]["status"] = "passed" + record.update(status="ready", stage="complete") + except BaseException as exc: + record.update(status="failed", error_type=type(exc).__name__) + if record.get("verifier_preflight", {}).get("status") == "pending": + record["verifier_preflight"]["status"] = "incomplete" + raise + finally: + record["cache_used"] = any( + a["restored"] for a in record.get("apt", []) + ) + self.logs_dir.mkdir(parents=True, exist_ok=True) + (self.logs_dir / "bootstrap.json").write_text( + json.dumps(record, indent=2) + "\n" + ) + + async def _install_agent(self, environment: BaseEnvironment) -> None: + if self._wheel_path is not None: + await environment.upload_file(self._wheel_path, self._install_target()) + self._bootstrap_record["wheel_sha256"] = hashlib.sha256( + self._wheel_path.read_bytes() + ).hexdigest() install_target = shlex.quote(self._install_target()) await self.exec_as_agent( environment, @@ -104,6 +172,49 @@ async def install(self, environment: BaseEnvironment) -> None: ), ) + async def _ensure_system_dependencies( + self, environment: BaseEnvironment + ) -> None: + """Install base dependencies, retrying a transient package-index 404. + + The retry only covers the system-package step, so it cannot replay any + model work; a non-404 failure or the final attempt propagates. + """ + if await self._get_system_package_manager(environment) == "apt-get": + packages = tuple( + package for dependency in _SYSTEM_DEPENDENCIES + for package in self.SYSTEM_PACKAGES[dependency].packages["apt-get"] + ) + await self._install_apt(environment, packages) + return + await self._retry_dependency_install( + lambda: self.ensure_system_dependencies(environment, _SYSTEM_DEPENDENCIES) + ) + + async def _install_apt(self, environment, packages) -> None: + await self._retry_dependency_install( + lambda: self._bootstrap_cache.install( + self, environment, packages, self._bootstrap_record, + ) + ) + + async def _retry_dependency_install(self, install) -> None: + total_attempts = len(_DEPENDENCY_RETRY_DELAYS) + 1 + for attempt, delay in enumerate((*_DEPENDENCY_RETRY_DELAYS, None)): + try: + await install() + return + except NonZeroAgentExitCodeError as error: + if delay is None or "404" not in str(error): + raise + self.logger.warning( + "Retrying system dependency installation after HTTP 404 " + "(attempt %s/%s); no model call has started.", + attempt + 1, + total_attempts, + ) + await asyncio.sleep(delay) + def _runtime_env(self) -> dict[str, str]: model_connection = self.model_connection env: dict[str, str] = {} @@ -155,6 +266,9 @@ async def run( @override def populate_context_post_run(self, context: AgentContext) -> None: + if self._bootstrap_record: + context.metadata = context.metadata or {} + context.metadata["bootstrap"] = self._bootstrap_record trajectory_path = self.logs_dir / "trajectory.json" try: trajectory = Trajectory.model_validate_json( @@ -193,6 +307,8 @@ def populate_context_post_run(self, context: AgentContext) -> None: usage_complete=extra.get("usage_complete"), cost_is_partial=extra.get("cost_is_partial"), known_cost_usd=extra.get("known_cost_usd"), + cost_is_estimated=extra.get("cost_is_estimated"), + estimated_cost_usd=extra.get("estimated_cost_usd"), missing_generation_ids=extra.get("missing_generation_ids"), ) if metrics is None: diff --git a/benchmarks/harbor/src/harbor_adapter/bootstrap.py b/benchmarks/harbor/src/harbor_adapter/bootstrap.py new file mode 100644 index 0000000..412315c --- /dev/null +++ b/benchmarks/harbor/src/harbor_adapter/bootstrap.py @@ -0,0 +1,183 @@ +"""Verified, reusable APT downloads and versioned verifier dependency profiles.""" + +from dataclasses import asdict, dataclass +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import shutil +import tempfile + + +def default_cache_dir() -> Path: + return Path( + os.environ.get("NANOPY_HARBOR_CACHE", ".cache/nanopy-harbor") + ).expanduser().resolve() + + +@dataclass(frozen=True) +class Package: + url: str + filename: str + size: int + sha256: str + + def __post_init__(self): + if not re.fullmatch(r"[A-Za-z0-9.+%~:_-]+\.deb", self.filename): + raise ValueError(f"Unsafe package filename: {self.filename!r}") + if type(self.size) is not int or self.size <= 0: + raise ValueError("Package size must be a positive integer") + if not re.fullmatch(r"[a-f0-9]{64}", self.sha256): + raise ValueError("Package requires a SHA256 digest") + + def verify(self, path: Path) -> None: + with path.open("rb") as source: + digest = hashlib.file_digest(source, "sha256").hexdigest() + if path.stat().st_size != self.size or digest != self.sha256: + raise ValueError(f"Package checksum/size mismatch: {self.filename}") + + +def parse_apt_plan(stdout: str) -> list[Package]: + packages = [] + for line in stdout.splitlines(): + if not line.startswith("'"): + continue + url, filename, size, digest = shlex.split(line) + if not digest.startswith("SHA256:"): + raise ValueError("APT download plan did not provide SHA256 hashes") + packages.append( + Package(url, filename, int(size), digest.removeprefix("SHA256:")) + ) + return packages + + +class AptCache: + """Key downloads by their APT-index hash, never by a mutable package name.""" + + def __init__(self, root: Path): + self.root = root + + def path(self, package: Package) -> Path: + return self.root / "apt" / package.sha256 / package.filename + + def store(self, package: Package, source: Path) -> None: + package.verify(source) + target = self.path(package) + target.parent.mkdir(parents=True, exist_ok=True) + # Atomic publication also supports concurrent trials sharing this cache. + with tempfile.TemporaryDirectory(dir=target.parent) as temporary: + staged = Path(temporary) / package.filename + shutil.copyfile(source, staged) + package.verify(staged) + staged.replace(target) + + def import_manifest(self, manifest: Path) -> int: + packages = [Package(**row) for row in json.loads(manifest.read_text())] + for package in packages: + package.verify(manifest.parent / "packages" / package.filename) + for package in packages: + self.store(package, manifest.parent / "packages" / package.filename) + return len(packages) + + async def install( + self, agent, environment, packages: tuple[str, ...], record: dict, + ) -> None: + """Restore only files selected by the current index; let APT install them. + + Download before installation so Docker's APT cleanup hooks cannot remove + successful downloads before they enter the shared cache. + """ + args = shlex.join(packages) + env = {"DEBIAN_FRONTEND": "noninteractive"} + await agent.exec_as_root(environment, command="apt-get update", env=env) + result = await agent.exec_as_root( + environment, + command=( + "apt-get -o Acquire::ForceHash=sha256 --print-uris " + f"-y --download-only install {args}" + ), + env=env, + ) + plan = parse_apt_plan(result.stdout) + attempt = { + "requested": list(packages), "packages": [asdict(p) for p in plan], + "restored": [], "saved": [], + } + record.setdefault("apt", []).append(attempt) + for package in plan: + cached = self.path(package) + if cached.exists(): + package.verify(cached) + await environment.upload_file( + cached, f"/var/cache/apt/archives/{package.filename}", + ) + attempt["restored"].append(package.filename) + await agent.exec_as_root( + environment, command=f"apt-get -y --download-only install {args}", env=env, + ) + with tempfile.TemporaryDirectory(prefix="nanopy-apt-") as temporary: + for package in plan: + if package.filename in attempt["restored"]: + continue + downloaded = Path(temporary) / package.filename + await environment.download_file( + source_path=f"/var/cache/apt/archives/{package.filename}", + target_path=downloaded, + ) + self.store(package, downloaded) + attempt["saved"].append(package.filename) + await agent.exec_as_root( + environment, command=f"apt-get install -y {args}", env=env, + ) + + +# Profiles describe dependencies only. Neither task solutions nor assertions are +# loaded during setup. A task revision change requires reviewing its profile. +QEMU_TASK = "terminal-bench/qemu-alpine-ssh" +QEMU_REF = "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28" +QEMU_PREFLIGHT = ( + "set -euo pipefail; " + "curl -LsSf https://astral.sh/uv/0.9.5/install.sh | sh; " + 'export PATH="$HOME/.local/bin:$PATH"; ' + "uv --version | grep -Eq '^uv 0\\.9\\.5( |$)'; " + "command -v sshpass; " + "uvx -p 3.13 -w pytest==8.4.1 -w pytest-json-ctrf==0.3.5 pytest --version" +) + + +@dataclass(frozen=True) +class VerifierProfile: + name: str + apt_packages: tuple[str, ...] + command: str + + +VERIFIER_PROFILES = { + (QEMU_TASK, QEMU_REF): VerifierProfile( + "qemu-alpine-ssh-v1", ("curl", "sshpass"), QEMU_PREFLIGHT, + ), +} + + +def verifier_profile( + logs_dir: Path, environment_name: str | None, +) -> VerifierProfile | None: + known_tasks = {name for name, _ in VERIFIER_PROFILES} + known_environment = environment_name in {name.split("/")[-1] for name in known_tasks} + config_path = logs_dir.parent / "config.json" + if not config_path.exists(): + if known_environment: + raise ValueError("Verifier preflight requires Harbor's pinned trial config") + return None + task = json.loads(config_path.read_text())["task"] + profile = VERIFIER_PROFILES.get((task.get("name"), task.get("ref"))) + # Local and Git datasets omit package name/ref. The environment identifies + # a known task, but cannot establish its reviewed revision by itself. + if profile is None and (task.get("name") in known_tasks or known_environment): + name = task.get("name") if task.get("name") in known_tasks else environment_name + raise ValueError( + f"Unknown {name} revision; review its verifier dependency profile" + ) + return profile diff --git a/benchmarks/harbor/src/harbor_adapter/verifier.py b/benchmarks/harbor/src/harbor_adapter/verifier.py new file mode 100644 index 0000000..7fd6e3f --- /dev/null +++ b/benchmarks/harbor/src/harbor_adapter/verifier.py @@ -0,0 +1,150 @@ +"""Harbor's script verifier with up to three attempts after a timeout.""" + +import asyncio +import json +import math +from pathlib import Path +import shlex +import uuid + +from harbor.models.task.config import TaskOS +from harbor.models.trial.config import TrialConfig +from harbor.models.trial.paths import EnvironmentPaths +from harbor.utils.scripts import build_execution_command +from harbor.verifier.verifier import Verifier + +_RUNNER = Path(__file__).with_name("_verifier_runner.py").read_text() + + +class _RetryEnvironment: + """Delegate Harbor operations, wrapping only the test-script execution.""" + + def __init__(self, environment, command, timeout_sec, max_attempts, logger): + self._environment = environment + self._command = command + self._timeout_sec = timeout_sec + self._max_attempts = max_attempts + self._logger = logger + + def __getattr__(self, name): + return getattr(self._environment, name) + + async def exec(self, *, command, **kwargs): + if command != self._command: + return await self._environment.exec(command=command, **kwargs) + control = f"/tmp/nanopy-verifier-{uuid.uuid4().hex}.pid" + invocation = shlex.join([ + "python3", "-c", _RUNNER, command, str(self._timeout_sec), + str(self._max_attempts), "/logs/verifier", control, + ]) + try: + result = await self._environment.exec(command=invocation, **kwargs) + except BaseException: + # Cancelling a Docker exec client alone can leave the command + # alive inside the container. Stop the runner and its test session. + try: + cleanup = await asyncio.wait_for(self._environment.exec( + command=shlex.join(["python3", "-c", _RUNNER, "cancel", control]), + user="root", + ), timeout=10) + if cleanup.return_code != 0: + raise RuntimeError(cleanup.stderr) + except Exception: + self._logger.exception("Failed to stop the verifier retry runner") + raise + if result.return_code != 0: + raise RuntimeError(f"Verifier retry runner failed: {result.stderr}") + record = json.loads(result.stdout) + if record["status"] == "timeout": + raise TimeoutError( + f"Verifier timed out after {self._max_attempts} attempts " + f"of {self._timeout_sec:g} seconds" + ) + return result + + +class RetryingVerifier(Verifier): + """Retry native Linux test-script timeouts without rerunning the agent. + + Select with ``--verifier harbor_adapter:RetryingVerifier`` and allow the + overall verification phase enough time via ``--verifier-timeout-multiplier + 4``. Each attempt retains the task's native verifier timeout. + """ + + def __init__(self, *, max_attempts=3, **kwargs): + if type(max_attempts) is not int or not 1 <= max_attempts <= 3: + raise ValueError("max_attempts must be an integer between 1 and 3") + super().__init__(**kwargs) + if self.environment.os != TaskOS.LINUX or self.step_name is not None: + raise ValueError("RetryingVerifier supports single-step Linux tasks") + self._max_attempts = max_attempts + self._attempt_timeout = self.task.config.verifier.timeout_sec + if not math.isfinite(self._attempt_timeout) or self._attempt_timeout <= 0: + raise ValueError("The task's verifier timeout must be finite and positive") + + async def verify(self): + # Harbor's outer watchdog covers the entire custom verifier. Reject an + # insufficient budget instead of silently losing the promised retries. + config = TrialConfig.model_validate_json(self.trial_paths.config_path.read_text()) + multiplier = config.verifier_timeout_multiplier + if multiplier is None: + multiplier = config.timeout_multiplier + base_timeout = ( + config.verifier.override_timeout_sec or self._attempt_timeout + ) + outer_timeout = min( + base_timeout, config.verifier.max_timeout_sec or float("inf"), + ) * multiplier + if outer_timeout <= self._attempt_timeout * self._max_attempts: + raise ValueError( + "Allow time for every verifier attempt and log collection: use " + "--verifier-timeout-multiplier 4 and remove conflicting timeout caps" + ) + _, tests_source_dir, host_test_path = self._resolve_tests() + paths = EnvironmentPaths.for_os(self.environment.os) + command = build_execution_command( + str(paths.tests_dir / host_test_path.relative_to(tests_source_dir).as_posix()), + stdout_path=str(paths.verifier_dir / "test-stdout.txt"), + task_os=self.environment.os, + ) + environment = self.environment + self.environment = _RetryEnvironment( + environment, command, self._attempt_timeout, self._max_attempts, self.logger, + ) + try: + return await super().verify() + finally: + self.environment = environment + # The standard verifier downloads logs only after successful exec. + # Preserve failed attempts too, even on non-mounted environments. + if not environment.capabilities.mounted: + try: + if self.include_logs or self.exclude_logs: + download = environment.download_dir_filtered( + source_dir=str(paths.verifier_dir), + target_dir=self.trial_paths.verifier_dir, + include=self.include_logs or None, + exclude=self.exclude_logs or None, + protect=["reward.txt", "reward.json", "retry-summary.json"], + ) + else: + download = environment.download_dir( + source_dir=str(paths.verifier_dir), + target_dir=self.trial_paths.verifier_dir, + ) + await asyncio.wait_for(download, timeout=30) + except Exception: + # Collection failure must not hide the original timeout, + # cancellation, or reward parsing error. + self.logger.exception("Failed to collect verifier attempt logs") + if self.include_logs or self.exclude_logs: + # Harbor protects exact file paths only. Download every + # archived attempt separately, regardless of log filters + # or failure to collect the current attempt's logs. + try: + await asyncio.wait_for(environment.download_dir( + source_dir=str(paths.verifier_dir / "attempts"), + target_dir=self.trial_paths.verifier_dir / "attempts", + ), timeout=30) + except Exception: + self.logger.exception("Failed to collect archived verifier attempts") diff --git a/benchmarks/harbor/tests/test_adapter.py b/benchmarks/harbor/tests/test_adapter.py index a4c5a4a..3c59e18 100644 --- a/benchmarks/harbor/tests/test_adapter.py +++ b/benchmarks/harbor/tests/test_adapter.py @@ -6,6 +6,7 @@ from types import SimpleNamespace import pytest +from harbor.agents.installed.base import NonZeroAgentExitCodeError from harbor.models.agent.context import AgentContext from harbor_adapter import NanoPyCodeAgent @@ -103,6 +104,79 @@ def test_install_source_rejects_a_revision_coerced_to_a_number(tmp_path): make_adapter(tmp_path, git_ref=float("inf")) +def test_setup_retries_a_transient_404_and_then_succeeds(tmp_path, monkeypatch): + import harbor_adapter.adapter as adapter_module + + adapter = make_adapter(tmp_path) + attempts = [] + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + async def flaky(environment, dependencies): + attempts.append(dependencies) + if len(attempts) < 3: + raise NonZeroAgentExitCodeError( + "Command failed (exit 100): apt-get install -y curl\n" + "E: Failed to fetch http://deb.debian.org/.../curl.deb 404 Not Found" + ) + + monkeypatch.setattr(adapter, "ensure_system_dependencies", flaky) + monkeypatch.setattr(adapter_module.asyncio, "sleep", fake_sleep) + + asyncio.run(adapter._ensure_system_dependencies(RecordingEnvironment())) + + assert len(attempts) == 3 + assert sleeps == [2.0, 4.0] + + +def test_setup_does_not_retry_a_non_404_failure(tmp_path, monkeypatch): + import harbor_adapter.adapter as adapter_module + + adapter = make_adapter(tmp_path) + attempts = [] + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + async def failing(environment, dependencies): + attempts.append(dependencies) + raise NonZeroAgentExitCodeError("Command failed (exit 1): false") + + monkeypatch.setattr(adapter, "ensure_system_dependencies", failing) + monkeypatch.setattr(adapter_module.asyncio, "sleep", fake_sleep) + + with pytest.raises(NonZeroAgentExitCodeError): + asyncio.run(adapter._ensure_system_dependencies(RecordingEnvironment())) + + assert len(attempts) == 1 + assert sleeps == [] + + +def test_setup_gives_up_after_the_last_retry(tmp_path, monkeypatch): + import harbor_adapter.adapter as adapter_module + + adapter = make_adapter(tmp_path) + attempts = [] + + async def fake_sleep(delay): + pass + + async def always_404(environment, dependencies): + attempts.append(dependencies) + raise NonZeroAgentExitCodeError("E: 404 Not Found") + + monkeypatch.setattr(adapter, "ensure_system_dependencies", always_404) + monkeypatch.setattr(adapter_module.asyncio, "sleep", fake_sleep) + + with pytest.raises(NonZeroAgentExitCodeError): + asyncio.run(adapter._ensure_system_dependencies(RecordingEnvironment())) + + assert len(attempts) == 3 + + def test_run_pipes_the_instruction_and_forwards_anthropic_configuration(tmp_path): instruction = 'fix "quoted" input; echo $TOKEN\nthen run the tests' adapter = make_adapter( diff --git a/benchmarks/harbor/tests/test_atif_compatibility.py b/benchmarks/harbor/tests/test_atif_compatibility.py index 4456078..d4ca37b 100644 --- a/benchmarks/harbor/tests/test_atif_compatibility.py +++ b/benchmarks/harbor/tests/test_atif_compatibility.py @@ -1,11 +1,15 @@ """Compatibility checks against Harbor's pinned ATIF-v1.7 validator.""" +import json from pathlib import Path import pytest +from harbor.models.agent.context import AgentContext from harbor.utils.trajectory_validator import TrajectoryValidator from nanopycodeagent.atif import project_atif +from nanopycodeagent.cost import estimated_cost, usage_cost from nanopycodeagent.event_journal import EventJournal, NativeEvent +from harbor_adapter import NanoPyCodeAgent def test_projector_output_passes_harbor_atif_validator(): @@ -16,6 +20,54 @@ def test_projector_output_passes_harbor_atif_validator(): assert validator.validate(trajectory), validator.get_errors() +@pytest.mark.parametrize("other_kind", [None, "estimated", "provider_reported", "pending"]) +def test_estimated_costs_survive_projection_validation_and_harbor_context(tmp_path, other_kind): + fixture = Path(__file__).parent / "fixtures" / "atif-journal-v1.jsonl" + usage = {"input_tokens": 1000, "output_tokens": 1000} + with EventJournal.create("run-estimated", directory=tmp_path) as journal: + for entry in EventJournal.replay(fixture): + payload = entry.payload + if entry.type == "model.completed": + payload = payload | { + "generation_id": None, + "cost": estimated_cost("deepseek-flash", usage), + } + journal.append(NativeEvent(entry.type, payload)) + if entry.type == "model.completed" and other_kind is not None: + other_cost = { + "estimated": estimated_cost("deepseek-flash", usage), + "provider_reported": usage_cost({"cost": "0.002"}), + "pending": {"status": "pending", "source": "provider_generation"}, + }[other_kind] + journal.append(NativeEvent("model.completed", payload | { + "model_call_id": "model-2", "generation_id": "generation-2", + "message_id": "msg-2", "stop_reason": "end_turn", + "content": [{"type": "text", "text": "done"}], + "tool_calls": [], "cost": other_cost, + })) + + trajectory = project_atif(EventJournal.replay(journal.path)) + validator = TrajectoryValidator() + assert validator.validate(trajectory), validator.get_errors() + (tmp_path / "trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8") + context = AgentContext() + NanoPyCodeAgent(logs_dir=tmp_path).populate_context_post_run(context) + + diagnostic = context.metadata["trajectory"] + assert diagnostic["cost_is_estimated"] is True + assert diagnostic["estimated_cost_usd"] == (0.003 if other_kind == "estimated" else 0.0015) + if other_kind == "pending": + assert context.cost_usd is None + assert diagnostic["status"] == "partial" + assert diagnostic["known_cost_usd"] == 0.0015 + assert diagnostic["missing_generation_ids"] == ["generation-2"] + else: + assert context.cost_usd == { + None: 0.0015, "estimated": 0.003, "provider_reported": 0.0035, + }[other_kind] + assert diagnostic["status"] == "complete" + + def test_time_budget_v4_passes_harbor_atif_validator(tmp_path): fixture = Path(__file__).parent / "fixtures" / "atif-journal-v1.jsonl" note = "[time budget] Stop investigating now. Write your best answer to the output file." diff --git a/benchmarks/harbor/tests/test_bootstrap.py b/benchmarks/harbor/tests/test_bootstrap.py new file mode 100644 index 0000000..7c5ff81 --- /dev/null +++ b/benchmarks/harbor/tests/test_bootstrap.py @@ -0,0 +1,229 @@ +"""Cache integrity and setup-before-model contracts for benchmark bootstrap.""" + +import asyncio +from dataclasses import asdict +import hashlib +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from harbor.models.trial.config import TrialConfig + +from harbor_adapter import NanoPyCodeAgent +from harbor_adapter.bootstrap import AptCache, Package, QEMU_REF, QEMU_TASK, parse_apt_plan, verifier_profile + + +def package(data=b"verified deb", filename="curl_1_amd64.deb"): + return Package("https://deb.example/" + filename, filename, len(data), hashlib.sha256(data).hexdigest()) + + +def apt_line(item): + return f"'{item.url}' {item.filename} {item.size} SHA256:{item.sha256}" + + +class AptEnvironment: + def __init__(self, item, data): + self.item = item + self.data = data + self.uploaded = [] + self.downloaded = [] + self.commands = [] + + async def upload_file(self, source, target): + self.uploaded.append((Path(source).read_bytes(), target)) + + async def download_file(self, source_path, target_path): + self.downloaded.append(source_path) + Path(target_path).write_bytes(self.data) + + async def exec_as_root(self, environment, command, **kwargs): + self.commands.append(command) + return SimpleNamespace(stdout=apt_line(self.item) if "--print-uris" in command else "") + + +def test_restores_only_the_current_index_hash_and_saves_downloads(tmp_path): + cache = AptCache(tmp_path) + data = b"current package" + item = package(data) + stale = package(b"old package") + source = tmp_path / "download.deb" + source.write_bytes(b"old package") + cache.store(stale, source) + first = AptEnvironment(item, data) + record = {} + asyncio.run(cache.install(first, first, ("curl",), record)) + assert first.uploaded == [] + assert len(first.downloaded) == 1 + assert cache.path(item).read_bytes() == data + assert record["apt"][0]["saved"] == [item.filename] + assert first.commands[-1] == "apt-get install -y curl" + + second = AptEnvironment(item, data) + record = {} + asyncio.run(cache.install(second, second, ("curl",), record)) + assert second.uploaded == [(data, "/var/cache/apt/archives/" + item.filename)] + assert second.downloaded == [] + assert record["apt"][0]["restored"] == [item.filename] + + +def test_corrupt_cache_fails_before_upload_or_install(tmp_path): + item = package() + cache = AptCache(tmp_path) + cache.path(item).parent.mkdir(parents=True) + cache.path(item).write_bytes(b"tampered") + environment = AptEnvironment(item, b"verified deb") + with pytest.raises(ValueError, match="checksum/size"): + asyncio.run(cache.install(environment, environment, ("curl",), {})) + assert not environment.uploaded + assert not any(command.startswith("apt-get install") for command in environment.commands) + + +def test_invalid_download_is_never_published_or_installed(tmp_path): + item = package() + cache = AptCache(tmp_path) + environment = AptEnvironment(item, b"corrupt response") + with pytest.raises(ValueError, match="checksum/size"): + asyncio.run(cache.install(environment, environment, ("curl",), {})) + assert not cache.path(item).exists() + assert not any(command.startswith("apt-get install") for command in environment.commands) + + +def test_import_validates_all_packages_before_publishing(tmp_path): + source = tmp_path / "old" + (source / "packages").mkdir(parents=True) + first = package() + second = package(b"another", "git_1_amd64.deb") + (source / "packages" / first.filename).write_bytes(b"verified deb") + (source / "packages" / second.filename).write_bytes(b"bad") + manifest = source / "manifest.json" + manifest.write_text(json.dumps([asdict(first), asdict(second)])) + cache = AptCache(tmp_path / "cache") + with pytest.raises(ValueError, match="checksum/size"): + cache.import_manifest(manifest) + assert not cache.root.exists() + (source / "packages" / second.filename).write_bytes(b"another") + assert cache.import_manifest(manifest) == 2 + assert cache.path(second).read_bytes() == b"another" + + +@pytest.mark.parametrize("filename", ["../escape.deb", "/tmp/escape.deb", "$(id).deb"]) +def test_manifest_rejects_unsafe_filenames(filename): + with pytest.raises(ValueError, match="Unsafe"): + package(filename=filename) + + +def test_apt_parser_requires_strong_hashes_and_handles_epoch_filenames(): + item = package(filename="git_1%3a2.30.2-1+deb11u5_amd64.deb") + assert parse_apt_plan("Reading package lists...\n" + apt_line(item)) == [item] + with pytest.raises(ValueError, match="SHA256"): + parse_apt_plan(apt_line(item).replace("SHA256:", "MD5Sum:")) + + +def write_task(logs_dir, ref=QEMU_REF): + logs_dir.mkdir(parents=True) + (logs_dir.parent / "config.json").write_text(json.dumps({"task": {"name": QEMU_TASK, "ref": ref}})) + + +def test_profile_is_scoped_to_reviewed_task_revision(tmp_path): + logs_dir = tmp_path / "agent" + write_task(logs_dir) + assert verifier_profile(logs_dir, "qemu-alpine-ssh").name == "qemu-alpine-ssh-v1" + (logs_dir.parent / "config.json").write_text(json.dumps({"task": {"name": QEMU_TASK, "ref": "latest"}})) + with pytest.raises(ValueError, match="Unknown"): + verifier_profile(logs_dir, "qemu-alpine-ssh") + + +@pytest.mark.parametrize("task", [ + {"path": "/tasks/qemu-alpine-ssh"}, + {"path": "/dataset/qemu-alpine-ssh", "source": "local-dataset"}, + { + "path": "tasks/qemu-alpine-ssh", "source": "git-dataset", + "git_url": "https://example.com/tasks.git", "git_commit_id": "a" * 40, + }, +], ids=["local-path", "local-dataset", "git-dataset"]) +def test_unresolved_qemu_revision_stops_setup_before_install(tmp_path, monkeypatch, task): + logs_dir = tmp_path / "agent" + logs_dir.mkdir() + (tmp_path / "config.json").write_text(TrialConfig(task=task).model_dump_json()) + adapter = NanoPyCodeAgent(logs_dir=logs_dir) + dependencies = AsyncMock() + install = AsyncMock() + monkeypatch.setattr(adapter, "_ensure_system_dependencies", dependencies) + monkeypatch.setattr(adapter, "_install_agent", install) + + with pytest.raises(ValueError, match="Unknown .*qemu-alpine-ssh revision"): + asyncio.run(adapter.install(SimpleNamespace(environment_name="qemu-alpine-ssh"))) + + dependencies.assert_not_awaited() + install.assert_not_awaited() + report = json.loads((logs_dir / "bootstrap.json").read_text()) + assert report["status"] == "failed" + assert report["stage"] == "profile" + + +def test_known_environment_requires_trial_config(tmp_path): + with pytest.raises(ValueError, match="pinned trial config"): + verifier_profile(tmp_path / "agent", "qemu-alpine-ssh") + + +@pytest.mark.parametrize("has_config", [False, True]) +def test_unregistered_task_does_not_require_a_profile(tmp_path, has_config): + if has_config: + config = TrialConfig(task={"path": "/tasks/other-task"}) + (tmp_path / "config.json").write_text(config.model_dump_json()) + assert verifier_profile(tmp_path / "agent", "other-task") is None + + +def test_preflight_failure_stops_setup_and_records_the_failing_stage(tmp_path, monkeypatch): + logs_dir = tmp_path / "agent" + write_task(logs_dir) + adapter = NanoPyCodeAgent(logs_dir=logs_dir) + calls = [] + + async def dependencies(*args): + calls.append("system") + + async def install(*args): + calls.append("agent_install") + + async def apt(*args): + calls.append("verifier_packages") + + async def fail(*args, command): + calls.append("verifier_preflight") + assert "pytest --version" in command + assert "/tests" not in command + raise RuntimeError("dependency download failed") + + monkeypatch.setattr(adapter, "_ensure_system_dependencies", dependencies) + monkeypatch.setattr(adapter, "_install_agent", install) + monkeypatch.setattr(adapter, "_install_apt", apt) + monkeypatch.setattr(adapter, "exec_as_agent", fail) + with pytest.raises(RuntimeError, match="dependency download"): + asyncio.run(adapter.install(SimpleNamespace(environment_name="qemu-alpine-ssh"))) + assert calls == ["system", "agent_install", "verifier_packages", "verifier_preflight"] + report = json.loads((logs_dir / "bootstrap.json").read_text()) + assert report["status"] == "failed" + assert report["stage"] == "verifier_dependencies" + assert report["verifier_preflight"]["status"] == "incomplete" + + +def test_local_wheel_is_uploaded_and_logged(tmp_path, monkeypatch): + wheel = tmp_path / "nanopycodeagent-0.1-py3-none-any.whl" + wheel.write_bytes(b"wheel contents") + adapter = NanoPyCodeAgent(logs_dir=tmp_path / "logs", wheel_path=str(wheel)) + environment = AptEnvironment(package(), b"") + calls = [] + + async def execute(*args, **kwargs): + calls.append(kwargs["command"]) + + monkeypatch.setattr(adapter, "exec_as_agent", execute) + asyncio.run(adapter._install_agent(environment)) + assert environment.uploaded == [(b"wheel contents", "/tmp/" + wheel.name)] + assert "uv tool install --force /tmp/" + wheel.name in calls[0] + assert adapter._bootstrap_record["wheel_sha256"] == hashlib.sha256(wheel.read_bytes()).hexdigest() + with pytest.raises(ValueError, match="mutually exclusive"): + NanoPyCodeAgent(logs_dir=tmp_path, wheel_path=str(wheel), git_ref="main") diff --git a/benchmarks/harbor/tests/test_verifier.py b/benchmarks/harbor/tests/test_verifier.py new file mode 100644 index 0000000..075c677 --- /dev/null +++ b/benchmarks/harbor/tests/test_verifier.py @@ -0,0 +1,324 @@ +"""Execute the retry runner with real processes and Harbor's reward parser.""" + +import asyncio +import json +import os +from pathlib import Path +import shlex +import shutil +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from harbor.environments.base import ExecResult +from harbor.models.task.config import TaskOS +from harbor.models.task.task import Task +from harbor.models.trial.config import TrialConfig, VerifierConfig +from harbor.models.trial.paths import TrialPaths +from harbor.trial.single_step import SingleStepTrial +from harbor.trial.trial import Trial +from harbor.trial.errors import VerifierTimeoutError +from harbor.utils.path_filter import filter_paths_by_patterns +from harbor.verifier.factory import VerifierFactory +from harbor.verifier.verifier import RewardFileNotFoundError, VerifierOutputParseError + +from harbor_adapter import RetryingVerifier + + +class LocalEnvironment: + """Map container paths locally while executing the real runner and scripts.""" + + os = TaskOS.LINUX + + def __init__(self, task, paths, mounted): + self.task = task + self.paths = paths + self.capabilities = SimpleNamespace(mounted=mounted) + self.logs = paths.verifier_dir if mounted else paths.trial_dir / "remote-logs" + self.logs.mkdir(exist_ok=True) + self.uploads = 0 + self.downloads = 0 + self.processes = [] + + async def upload_dir(self, **kwargs): + self.uploads += 1 + + async def download_dir(self, *, source_dir, target_dir): + self.downloads += 1 + source = self.logs / Path(source_dir).relative_to("/logs/verifier") + shutil.copytree(source, target_dir, dirs_exist_ok=True) + + async def download_dir_filtered(self, *, source_dir, target_dir, include=None, exclude=None, protect=None): + source = self.logs / Path(source_dir).relative_to("/logs/verifier") + paths = [path.relative_to(source).as_posix() for path in source.rglob("*") if path.is_file()] + selected = set(filter_paths_by_patterns(paths, include=include, exclude=exclude)) + # Harbor's protected entries are exact relative paths, never globs. + selected.update(set(protect or []) & set(paths)) + for path in selected: + target = Path(target_dir) / path + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source / path, target) + + async def exec(self, *, command, env=None, **kwargs): + if command.startswith("chmod +x"): + self.task.paths.test_path.chmod(0o755) + return ExecResult(return_code=0) + args = shlex.split(command) + assert args[:2] == ["python3", "-c"] + if args[3] != "cancel": + args[3] = args[3].replace("/tests/", str(self.task.paths.tests_dir) + "/") + args[3] = args[3].replace("/logs/verifier", str(self.logs)) + args[-2] = str(self.logs) + args[-1] = str(self.paths.trial_dir / Path(args[-1]).name) + process = await asyncio.create_subprocess_exec( + *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + env={**os.environ, "LOGS": str(self.logs), + "STATE": str(self.paths.trial_dir / "count"), + "CHILD_PID": str(self.paths.trial_dir / "child.pid"), **(env or {})}, + ) + self.processes.append(process) + stdout, stderr = await process.communicate() + return ExecResult(return_code=process.returncode, stdout=stdout.decode(), stderr=stderr.decode()) + + +def make_verifier(tmp_path, script, *, mounted=True, max_attempts=3, include_logs=None, exclude_logs=None): + task_dir = tmp_path / "task" + (task_dir / "tests").mkdir(parents=True) + (task_dir / "instruction.md").write_text("Write the solution.") + (task_dir / "task.toml").write_text("[verifier]\ntimeout_sec = 0.3\n") + (task_dir / "tests" / "test.sh").write_text("#!/bin/bash\nset -eu\n" + script) + task = Task(task_dir) + paths = TrialPaths(trial_dir=tmp_path / "trial") + paths.mkdir() + config = TrialConfig( + task={"path": task_dir}, verifier_timeout_multiplier=4, + verifier=VerifierConfig( + import_path="harbor_adapter:RetryingVerifier", + kwargs={"max_attempts": max_attempts}, + include_logs=include_logs or [], exclude_logs=exclude_logs or [], + ), + ) + paths.config_path.write_text(config.model_dump_json()) + environment = LocalEnvironment(task, paths, mounted) + verifier = VerifierFactory.create_verifier_from_config( + config.verifier, task=task, trial_paths=paths, environment=environment, + ) + return verifier, environment, config + + +def summary(verifier): + return json.loads((verifier.trial_paths.verifier_dir / "retry-summary.json").read_text()) + + +@pytest.mark.parametrize("mounted", [True, False]) +def test_timeout_retries_preserve_logs_clear_rewards_and_run_agent_once(tmp_path, mounted): + verifier, environment, config = make_verifier(tmp_path, ''' +count=$(cat "$STATE" 2>/dev/null || echo 0) +count=$((count + 1)) +echo "$count" > "$STATE" +echo "attempt $count" +if [ "$count" -lt 3 ]; then + echo '{"reward": 1}' > "$LOGS/reward.json" + touch "$LOGS/stale.log" + sleep 10 +fi +test ! -e "$LOGS/stale.log" +test ! -e "$LOGS/reward.json" +echo 0 > "$LOGS/reward.txt" +''', mounted=mounted) + # Exercise Harbor's single-step lifecycle with a local verifier and its + # real outer watchdog. No installation, Docker, or model call is needed. + trial = object.__new__(SingleStepTrial) + trial.task = verifier.task + trial.config = config + for name in ("_run_agent", "_upload_agent_logs", "_collect_artifacts", "_stop_agent_environment"): + setattr(trial, name, AsyncMock()) + results = [] + + async def verify(): + results.append(await asyncio.wait_for( + verifier.verify(), timeout=Trial._compute_verifier_timeout_sec(trial), + )) + + trial._run_verifier = verify + asyncio.run(trial._run()) + + trial._run_agent.assert_awaited_once() + assert results[0].rewards == {"reward": 0.0} + assert environment.uploads == 1 + assert [item["status"] for item in summary(verifier)["attempts"]] == [ + "timeout", "timeout", "completed", + ] + logs = verifier.trial_paths.verifier_dir + for attempt in (1, 2, 3): + assert (logs / "attempts" / str(attempt) / "test-stdout.txt").read_text() == f"attempt {attempt}\n" + assert (logs / "attempts/1/reward.json").exists() + assert not (logs / "reward.json").exists() + + +@pytest.mark.parametrize("reward", [0, 1]) +def test_any_valid_score_stops_after_first_attempt(tmp_path, reward): + verifier, _, _ = make_verifier(tmp_path, f'echo {reward} > "$LOGS/reward.txt"\n') + assert asyncio.run(verifier.verify()).rewards == {"reward": reward} + assert len(summary(verifier)["attempts"]) == 1 + + +@pytest.mark.parametrize("mounted", [True, False]) +def test_three_timeouts_leave_no_canonical_reward_and_keep_all_logs(tmp_path, mounted): + verifier, environment, _ = make_verifier(tmp_path, ''' +echo started +echo 1 > "$LOGS/reward.txt" +sleep 10 +''', mounted=mounted) + with pytest.raises(TimeoutError, match="3 attempts"): + asyncio.run(verifier.verify()) + assert len(summary(verifier)["attempts"]) == 3 + assert all(item["status"] == "timeout" for item in summary(verifier)["attempts"]) + assert not verifier.trial_paths.reward_text_path.exists() + assert (verifier.trial_paths.verifier_dir / "attempts/3/test-stdout.txt").read_text() == "started\n" + assert environment.downloads == (0 if mounted else 1) + + +@pytest.mark.parametrize("filters", [ + {"include_logs": ["test-stdout.txt"]}, + {"exclude_logs": ["attempts/*", "retry-summary.json", "reward.txt", "details/*"]}, +], ids=["include-current-stdout", "exclude-archives"]) +@pytest.mark.parametrize("exhausted", [False, True], ids=["completed", "exhausted"]) +def test_log_filters_preserve_every_attempt_on_non_mounted_environments(tmp_path, filters, exhausted): + script = ''' +count=$(cat "$STATE" 2>/dev/null || echo 0) +count=$((count + 1)) +echo "$count" > "$STATE" +echo "attempt $count" +mkdir -p "$LOGS/details" +echo "detail $count" > "$LOGS/details/check.txt" +echo 1 > "$LOGS/reward.txt" +''' + script += 'sleep 10\n' if exhausted else 'if [ "$count" -lt 3 ]; then sleep 10; fi\n' + verifier, environment, _ = make_verifier(tmp_path, script, mounted=False, **filters) + if exhausted: + with pytest.raises(TimeoutError, match="3 attempts"): + asyncio.run(verifier.verify()) + else: + assert asyncio.run(verifier.verify()).rewards == {"reward": 1} + + # Only downloaded files survive environment teardown. + shutil.rmtree(environment.logs) + logs = verifier.trial_paths.verifier_dir + assert [item["status"] for item in summary(verifier)["attempts"]] == [ + "timeout", "timeout", "timeout" if exhausted else "completed", + ] + for attempt in (1, 2, 3): + archive = logs / "attempts" / str(attempt) + assert (archive / "test-stdout.txt").read_text() == f"attempt {attempt}\n" + assert (archive / "details/check.txt").read_text() == f"detail {attempt}\n" + assert (archive / "reward.txt").read_text() == "1\n" + assert verifier.trial_paths.reward_text_path.exists() is (not exhausted) + assert not (logs / "details/check.txt").exists() + + +@pytest.mark.parametrize(("script", "error"), [ + ("exit 124\n", RewardFileNotFoundError), + ('echo invalid > "$LOGS/reward.txt"\n', VerifierOutputParseError), +]) +def test_non_timeout_failures_are_not_retried(tmp_path, script, error): + verifier, _, _ = make_verifier(tmp_path, script) + with pytest.raises(error): + asyncio.run(verifier.verify()) + assert len(summary(verifier)["attempts"]) == 1 + assert summary(verifier)["attempts"][0]["status"] == "completed" + + +def process_is_running(pid): + try: + stat = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + except FileNotFoundError: + return False + return stat[0] != "Z" + + +def test_timeout_stops_children_in_additional_process_groups(tmp_path): + verifier, _, _ = make_verifier(tmp_path, ''' +timeout 30 bash -c 'echo $$ > "$CHILD_PID"; sleep 30' & +wait +''', max_attempts=1) + with pytest.raises(TimeoutError, match="1 attempts"): + asyncio.run(verifier.verify()) + pid = int((verifier.trial_paths.trial_dir / "child.pid").read_text()) + assert not process_is_running(pid) + + +def test_cancellation_stops_the_runner_and_never_retries(tmp_path): + verifier, environment, _ = make_verifier(tmp_path, 'echo $$ > "$CHILD_PID"\nsleep 30\n') + + async def cancel(): + verification = asyncio.create_task(verifier.verify()) + pid_file = verifier.trial_paths.trial_dir / "child.pid" + async with asyncio.timeout(5): + while not pid_file.exists(): + await asyncio.sleep(0.01) + verification.cancel() + with pytest.raises(asyncio.CancelledError): + await verification + for process in environment.processes: + await asyncio.wait_for(process.wait(), 2) + + asyncio.run(cancel()) + assert len(summary(verifier)["attempts"]) == 1 + assert summary(verifier)["attempts"][0]["status"] == "cancelled" + pid = int((verifier.trial_paths.trial_dir / "child.pid").read_text()) + assert not process_is_running(pid) + + +@pytest.mark.parametrize("value", [0, 4, -1, True, "3", 1.5]) +def test_invalid_attempt_limits_are_rejected(value): + with pytest.raises(ValueError, match="max_attempts"): + RetryingVerifier(max_attempts=value) + + +def test_outer_watchdog_must_allow_all_attempts(tmp_path): + verifier, environment, config = make_verifier(tmp_path, "exit 0\n") + config.verifier_timeout_multiplier = 1 + verifier.trial_paths.config_path.write_text(config.model_dump_json()) + with pytest.raises(ValueError, match="verifier-timeout-multiplier 4"): + asyncio.run(verifier.verify()) + assert environment.uploads == 0 + + +def test_log_download_failure_preserves_harbor_timeout_error(tmp_path, caplog): + verifier, environment, config = make_verifier( + tmp_path, "sleep 10\n", mounted=False, max_attempts=1, + ) + environment.download_dir = AsyncMock(side_effect=OSError("download failed")) + trial = object.__new__(SingleStepTrial) + trial.task = verifier.task + trial.config = config + trial._verifier_timeout_sec = Trial._compute_verifier_timeout_sec(trial) + trial._result = SimpleNamespace(verifier=None) + trial._emit = AsyncMock() + trial._run_shared_verifier = lambda **kwargs: verifier.verify() + + with pytest.raises(VerifierTimeoutError): + asyncio.run(trial._run_verifier()) + assert "Failed to collect verifier attempt logs" in caplog.text + environment.download_dir.assert_awaited_once() + + +def test_filtered_download_failure_still_collects_attempt_archives(tmp_path, caplog): + verifier, environment, _ = make_verifier( + tmp_path, 'echo started\nsleep 10\n', mounted=False, max_attempts=1, + include_logs=["test-stdout.txt"], + ) + environment.download_dir_filtered = AsyncMock(side_effect=OSError("download failed")) + with pytest.raises(TimeoutError, match="1 attempts"): + asyncio.run(verifier.verify()) + assert "Failed to collect verifier attempt logs" in caplog.text + archive = verifier.trial_paths.verifier_dir / "attempts/1/test-stdout.txt" + assert archive.read_text() == "started\n" + + +def test_outer_timeout_cap_is_applied_before_multiplier(tmp_path): + verifier, _, config = make_verifier(tmp_path, 'echo 1 > "$LOGS/reward.txt"\n') + config.verifier.max_timeout_sec = 0.3 + verifier.trial_paths.config_path.write_text(config.model_dump_json()) + assert asyncio.run(verifier.verify()).rewards == {"reward": 1} diff --git a/benchmarks/harbor/tests/test_workflow.py b/benchmarks/harbor/tests/test_workflow.py new file mode 100644 index 0000000..8d61ad0 --- /dev/null +++ b/benchmarks/harbor/tests/test_workflow.py @@ -0,0 +1,90 @@ +"""Standard workflow policy and score/infra separation.""" + +import json + +import pytest + +from harbor_adapter.__main__ import standard_config, summarize + + +def config(): + return { + "tasks": [{"name": "terminal-bench/test", "ref": "sha256:abc"}], + "agents": [{"name": "harbor_adapter:NanoPyCodeAgent", "model_name": "test/model"}], + } + + +def test_standard_policy_enables_native_verifier_retries_without_agent_retries(): + result = standard_config(config()) + assert result.n_attempts == 1 + assert result.retry.max_retries == 0 + assert result.verifier.import_path == "harbor_adapter:RetryingVerifier" + assert result.verifier.kwargs == {"max_attempts": 3} + assert result.verifier_timeout_multiplier == 4 + + +def test_install_only_json_config_is_accepted_and_round_trips(): + result = standard_config(config() | {"install_only": True}) + assert result.install_only is True + assert result.verifier.disable is True + assert standard_config(json.loads(result.model_dump_json())) == result + + +def test_full_runs_still_require_verification(): + raw = config() | { + "verifier": {"import_path": "harbor_adapter:RetryingVerifier", "disable": True}, + } + with pytest.raises(ValueError, match="verification enabled"): + standard_config(raw) + + +@pytest.mark.parametrize("change", [ + {"retry": {"max_retries": 1}}, + {"verifier_timeout_multiplier": 1}, + {"verifier": {"disable": True}}, + {"verifier": {"import_path": "custom:Verifier"}}, + {"agents": [{"name": "custom:Agent", "model_name": "test/model"}]}, + {"agents": [{"name": "custom:Agent", "import_path": "harbor_adapter:NanoPyCodeAgent", "model_name": "test/model"}]}, + {"agents": []}, + {"agents": [{"name": "harbor_adapter:NanoPyCodeAgent", "model_name": "test/model", "kwargs": {"version": "old"}}]}, +]) +def test_conflicting_workflows_fail_before_a_run(change): + with pytest.raises(ValueError): + standard_config(config() | change) + + +def test_summary_does_not_turn_infrastructure_failure_into_zero(tmp_path): + failed = tmp_path / "failed" + failed.mkdir() + (failed / "result.json").write_text(json.dumps({"task_name": "qemu", "exception_info": {"exception_type": "AgentSetupTimeoutError"}, "verifier_result": None})) + scored = tmp_path / "scored" + scored.mkdir() + (scored / "result.json").write_text(json.dumps({"task_name": "qemu", "exception_info": None, "verifier_result": {"rewards": {"reward": 0.0}}})) + rows = summarize(tmp_path)["trials"] + assert rows[0]["exception_type"] == "AgentSetupTimeoutError" + assert rows[0]["rewards"] is None + assert rows[1]["status"] == "scored" + assert rows[1]["rewards"] == {"reward": 0.0} + + +def test_install_only_success_is_reported_without_a_score(tmp_path): + trial = tmp_path / "installed" + (trial / "agent").mkdir(parents=True) + (trial / "result.json").write_text(json.dumps({"task_name": "qemu"})) + (trial / "agent/bootstrap.json").write_text(json.dumps({"status": "ready", "cache_used": True})) + row, = summarize(tmp_path, install_only=True)["trials"] + assert row["status"] == "setup_passed" + assert row["rewards"] is None + assert row["bootstrap"]["cache_used"] is True + + +def test_exception_invalidates_a_partial_reward(tmp_path): + trial = tmp_path / "errored" + trial.mkdir() + (trial / "result.json").write_text(json.dumps({ + "exception_info": {"exception_type": "VerifierTimeoutError"}, + "verifier_result": {"rewards": {"reward": 0.0}}, + })) + row, = summarize(tmp_path)["trials"] + assert row["status"] == "execution_error" + assert row["rewards"] is None diff --git a/docs/changelogs/0.8.x.md b/docs/changelogs/0.8.x.md index 8107711..83c73b4 100644 --- a/docs/changelogs/0.8.x.md +++ b/docs/changelogs/0.8.x.md @@ -5,6 +5,23 @@ All notable changes in the **0.8.x** release series are documented here. ## [Unreleased] ### Added +- Standardize Harbor runs through a recorded working-tree wheel, a shared + SHA256-verified APT cache, and task-revision-specific dependency preflight. + Add a QEMU Alpine SSH profile and an install-only check before model spend; + preserve setup failures separately from scored results and test the workflow + contracts in CI. +- Add an opt-in Harbor `RetryingVerifier` that retries a timed-out Linux test + script up to three total attempts at the task's native timeout, reusing the + existing agent output. Preserve attempt logs, clear stale rewards, and stop + timed-out test sessions before retrying. Completed scores, including zero, + and non-timeout errors are not retried. +- Estimate model-call costs from token usage when a provider reports tokens + but no per-request cost, starting with the DeepSeek official + Anthropic-compatible endpoint. The estimate uses a static DeepSeek price + list, is marked `kind: estimated` so it stays distinct from a + provider-reported or reconciled cost, and is only used after a real + provider-reported cost is absent. DeepSeek exposes no cost or generation + endpoint, so the existing OpenRouter reconciliation cannot resolve it. - Configure per-reply generation budgets through `--max-tokens` or `ANTHROPIC_MAX_TOKENS`, including the user settings file and Harbor adapter. Record the effective limit in startup output, Event Journals, and ATIF @@ -25,6 +42,15 @@ All notable changes in the **0.8.x** release series are documented here. supported. ### Fixed +- Preserve estimated-cost classification and subtotals in ATIF and Harbor + summaries, while allowing generation billing lookups to replace estimates. +- Reject unresolved QEMU task revisions selected through local paths or + datasets, accept install-only mode from JSON configuration, and preserve + archived verifier attempts when Harbor filters downloaded logs. +- Retry the Harbor adapter's system-dependency installation a bounded number + of times when the package index returns a transient HTTP 404, instead of + failing agent setup immediately. Non-404 failures and the final attempt still + propagate. - Terminate a timed-out or interrupted bash command's process group on POSIX and other members of its session on Linux, including GNU `timeout` children in separate groups. Close output pipes without draining detached children diff --git a/src/nanopycodeagent/agent.py b/src/nanopycodeagent/agent.py index cb4ee32..d7773a9 100644 --- a/src/nanopycodeagent/agent.py +++ b/src/nanopycodeagent/agent.py @@ -42,6 +42,7 @@ from .atif import project_atif, write_atif from .bash_tool import run_bash from .cost import ( + estimated_cost, pending_cost, resolve_generation_cost, usage_cost, @@ -767,6 +768,10 @@ def _run_model_loop( ), "generation_id": generation_id, "cost": usage_cost(usage if isinstance(usage, dict) else None) + or estimated_cost( + str(getattr(message, "model", None) or model), + usage if isinstance(usage, dict) else None, + ) or pending_cost(generation_id), "duration_ms": (model_completed_ns - model_started_ns) / 1_000_000, "source_timestamp": utc_now(), @@ -836,7 +841,7 @@ def _reconcile_costs( *, max_seconds: float | None = None, ) -> list[JsonObject]: - """Append resolved OpenRouter costs without affecting the run outcome.""" + """Reconcile pending or estimated costs without affecting the run outcome.""" base_url = getattr(client, "base_url", "") credential = client.api_key or client.auth_token if not isinstance(credential, str) or not credential: @@ -862,7 +867,7 @@ def _reconcile_costs( not isinstance(generation_id, str) or generation_id in already_resolved or not isinstance(cost, dict) - or cost.get("status") != "pending" + or (cost.get("status") != "pending" and cost.get("kind") != "estimated") ): continue diagnostics: list[JsonObject] = [] diff --git a/src/nanopycodeagent/atif.py b/src/nanopycodeagent/atif.py index 9ae019b..9c5d9a1 100644 --- a/src/nanopycodeagent/atif.py +++ b/src/nanopycodeagent/atif.py @@ -271,6 +271,7 @@ def project_atif(entries: Sequence[JournalEntry]) -> JsonObject: if entry.type == "model.cost_resolved" } cost_states: list[tuple[str | None, Decimal | None]] = [] + estimated_costs: list[Decimal] = [] terminal: JournalEntry | None = None for entry in entries[1:]: payload = entry.payload @@ -390,6 +391,10 @@ def project_atif(entries: Sequence[JournalEntry]) -> JsonObject: metrics_extra = metrics.setdefault("extra", {}) assert isinstance(metrics_extra, dict) metrics_extra["cost_source"] = resolved_cost["source"] + if "kind" in resolved_cost: + metrics_extra["cost_kind"] = resolved_cost["kind"] + if resolved_cost.get("kind") == "estimated": + estimated_costs.append(amount) if generation_id is not None: metrics_extra["generation_id"] = generation_id if isinstance(cost, dict) or resolved_cost is not None: @@ -538,6 +543,13 @@ def project_atif(entries: Sequence[JournalEntry]) -> JsonObject: ] if missing: final_extra["missing_generation_ids"] = missing + if estimated_costs: + # Either the complete total or the known partial subtotal includes + # estimates. Keep their contribution separate from billed amounts. + final_extra = final_metrics.setdefault("extra", {}) + assert isinstance(final_extra, dict) + final_extra["cost_is_estimated"] = True + final_extra["estimated_cost_usd"] = float(sum(estimated_costs, Decimal(0))) return trajectory diff --git a/src/nanopycodeagent/cost.py b/src/nanopycodeagent/cost.py index ea54ffb..a30d7eb 100644 --- a/src/nanopycodeagent/cost.py +++ b/src/nanopycodeagent/cost.py @@ -15,6 +15,66 @@ DEFAULT_RETRY_DELAYS = (1.0, 2.0, 4.0, 8.0, 15.0) RETRYABLE_HTTP_STATUSES = frozenset({404, 408, 409, 429, 500, 502, 503, 504}) +# Static price lists in USD per million tokens for providers that return token +# usage but not a per-request cost. DeepSeek's official endpoint is the current +# case: it reports ``input_tokens``, ``output_tokens``, and cache token counts, +# but no ``cost`` field and no generation id, so the existing usage and +# generation reconciliation paths cannot resolve a cost. Update these numbers +# when the provider changes its pricing. DeepSeek's cache-write price is zero. +_TOKEN_PRICES: dict[str, dict[str, Decimal]] = { + "deepseek-flash": { + "input": Decimal("0.3"), + "output": Decimal("1.2"), + "cache_read": Decimal("0.006"), + "cache_write": Decimal("0"), + }, +} +_TOKENS_PER_PRICE_UNIT = Decimal(1_000_000) + + +def estimated_cost(model: object, usage: JsonObject | None) -> JsonObject | None: + """Estimate a USD cost from token usage for a priced, non-reporting model. + + Returns ``None`` when the model has no local price list or the usage does + not carry the integer token counts the estimate needs. The result is marked + ``kind: estimated`` so it stays distinguishable from a provider-reported or + reconciled cost. Call this only after :func:`usage_cost` fails, so an + explicit provider cost always wins. + """ + if not isinstance(model, str) or not isinstance(usage, dict): + return None + prices = _TOKEN_PRICES.get(model) + if prices is None: + return None + if "input_tokens" not in usage or "output_tokens" not in usage: + return None + counts: dict[str, int] = {} + for field in ( + "input_tokens", + "output_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + ): + value = usage.get(field, 0) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return None + counts[field] = value + amount = ( + Decimal(counts["input_tokens"]) * prices["input"] + + Decimal(counts["cache_read_input_tokens"]) * prices["cache_read"] + + Decimal(counts["cache_creation_input_tokens"]) * prices["cache_write"] + + Decimal(counts["output_tokens"]) * prices["output"] + ) / _TOKENS_PER_PRICE_UNIT + if not amount.is_finite() or amount < 0: + return None + return { + "status": "resolved", + "amount": str(amount), + "currency": "USD", + "source": "token_estimate.deepseek", + "kind": "estimated", + } + def generation_url(base_url: object) -> str | None: """Build the provider-local generation endpoint from an SDK base URL.""" diff --git a/tests/test_agent_events.py b/tests/test_agent_events.py index b83eb58..8c4fe0b 100644 --- a/tests/test_agent_events.py +++ b/tests/test_agent_events.py @@ -7,6 +7,7 @@ import pytest from nanopycodeagent import agent, settings +from nanopycodeagent.atif import project_atif from nanopycodeagent.event_journal import EventJournal from helpers import ( @@ -188,6 +189,74 @@ def resolved(base_url, generation_id, credential, **kwargs): ] +@pytest.mark.parametrize( + ("generation_id", "provider_cost", "billed_cost", "expected_cost", "is_estimated"), + [ + (None, None, None, 0.0015, True), + ("gen-deepseek", None, "0.00072", 0.00072, False), + ("gen-deepseek", None, "0", 0.0, False), + ("gen-deepseek", None, None, 0.0015, True), + ("gen-deepseek", "0.0008", None, 0.0008, False), + ("gen-deepseek", "0", None, 0.0, False), + ], +) +def test_deepseek_estimate_preserves_billing_reconciliation( + monkeypatch, generation_id, provider_cost, billed_cost, expected_cost, is_estimated +): + usage = {"input_tokens": 1000, "output_tokens": 1000} + if provider_cost is not None: + usage["cost"] = provider_cost + reply = FakeStream( + [text_block("done")], + model="deepseek-flash", + usage=SimpleNamespace(**usage), + response_headers={"x-generation-id": generation_id} if generation_id else {}, + ) + patch_client(monkeypatch, FakeClient(FakeMessages([reply]))) + lookups = [] + + def resolve(base_url, requested_id, credential, **kwargs): + lookups.append(requested_id) + if billed_cost is None: + return None + return { + "generation_id": requested_id, + "amount": billed_cost, + "currency": "USD", + "source": "provider_generation.total_cost", + } + + monkeypatch.setattr(agent, "resolve_generation_cost", resolve) + + assert agent.run_headless("fix it") == 0 + + should_reconcile = generation_id is not None and provider_cost is None + assert lookups == ([generation_id] if should_reconcile else []) + entries = EventJournal.replay(_only_journal_path()) + if should_reconcile: + assert entries[-1].payload["cost_reconciliation"][0]["status"] == ( + "resolved" if billed_cost is not None else "unresolved" + ) + trajectory = project_atif(entries) + metrics = trajectory["final_metrics"] + step_metrics = next( + step["metrics"] for step in trajectory["steps"] if step["source"] == "agent" + ) + assert step_metrics["cost_usd"] == expected_cost + assert metrics["total_cost_usd"] == expected_cost + extra = metrics.get("extra", {}) + assert extra.get("cost_is_estimated", False) is is_estimated + if is_estimated: + assert step_metrics["extra"]["cost_kind"] == "estimated" + assert extra["estimated_cost_usd"] == expected_cost + else: + assert step_metrics["extra"]["cost_source"] == ( + "provider_response.usage.cost" if provider_cost is not None + else "provider_generation.total_cost" + ) + assert "estimated_cost_usd" not in extra + + def test_unresolved_cost_diagnostics_are_persisted_on_run_terminal(monkeypatch): reply = FakeStream( [text_block("done")], diff --git a/tests/test_atif.py b/tests/test_atif.py index 25bbdcc..901ad59 100644 --- a/tests/test_atif.py +++ b/tests/test_atif.py @@ -2,6 +2,7 @@ import json import stat +from dataclasses import replace from pathlib import Path import pytest @@ -223,6 +224,48 @@ def test_resolved_and_missing_costs_project_as_partial_metrics(tmp_path): assert trajectory["final_metrics"]["total_cost_usd"] == 0.00072 +@pytest.mark.parametrize("estimated_amount", ["0", "0.0015"]) +@pytest.mark.parametrize("other_kind", ["estimated", "provider_reported", "pending"]) +def test_estimated_cost_summary_preserves_mixed_and_partial_costs( + tmp_path, estimated_amount, other_kind +): + entries = _journal_entries(tmp_path) + completed = next(entry for entry in entries if entry.type == "model.completed") + completed.payload["generation_id"] = None + completed.payload["cost"] = { + "status": "resolved", "amount": estimated_amount, "currency": "USD", + "source": "token_estimate.deepseek", "kind": "estimated", + } + other_cost = {"status": "pending", "source": "provider_generation"} + if other_kind != "pending": + other_cost = { + "status": "resolved", "amount": "0.002", "currency": "USD", + "source": "token_estimate.deepseek" if other_kind == "estimated" + else "provider_response.usage.cost", + "kind": other_kind, + } + entries.insert(-1, replace(completed, payload=completed.payload | { + "model_call_id": "model-2", "generation_id": "generation-2", "cost": other_cost, + })) + + trajectory = project_atif(entries) + metrics = trajectory["final_metrics"] + extra = metrics["extra"] + assert extra["cost_is_estimated"] is True + assert extra["estimated_cost_usd"] == pytest.approx( + float(estimated_amount) + (0.002 if other_kind == "estimated" else 0) + ) + assert trajectory["steps"][1]["metrics"]["extra"]["cost_kind"] == "estimated" + if other_kind == "pending": + assert "total_cost_usd" not in metrics + assert extra["known_cost_usd"] == float(estimated_amount) + assert extra["cost_is_partial"] is True + assert extra["missing_generation_ids"] == ["generation-2"] + else: + assert metrics["total_cost_usd"] == pytest.approx(float(estimated_amount) + 0.002) + assert "cost_is_partial" not in extra + + def test_cost_reconciliation_diagnostics_project_to_terminal_extra(tmp_path): entries = _journal_entries(tmp_path) entries[-1].payload["cost_reconciliation"] = [ diff --git a/tests/test_cost.py b/tests/test_cost.py index d827378..9d7fb22 100644 --- a/tests/test_cost.py +++ b/tests/test_cost.py @@ -4,6 +4,7 @@ import pytest from nanopycodeagent.cost import ( + estimated_cost, generation_url, pending_cost, resolve_generation_cost, @@ -39,6 +40,52 @@ def test_usage_cost_preserves_provider_reported_decimal(): } +def test_estimated_cost_prices_deepseek_flash_token_usage(): + assert estimated_cost( + "deepseek-flash", + { + "input_tokens": 5719, + "output_tokens": 2203, + "cache_read_input_tokens": 25600, + "cache_creation_input_tokens": 0, + }, + ) == { + "status": "resolved", + "amount": "0.0045129", + "currency": "USD", + "source": "token_estimate.deepseek", + "kind": "estimated", + } + + +def test_estimated_cost_defaults_absent_cache_counts_to_zero(): + # A minimal DeepSeek-style usage object still prices without cache fields. + assert estimated_cost( + "deepseek-flash", {"input_tokens": 1000, "output_tokens": 1000} + ) == { + "status": "resolved", + "amount": "0.0015", + "currency": "USD", + "source": "token_estimate.deepseek", + "kind": "estimated", + } + + +@pytest.mark.parametrize( + ("model", "usage"), + [ + ("unknown-model", {"input_tokens": 1, "output_tokens": 1}), + ("deepseek-flash", None), + ("deepseek-flash", {}), + ("deepseek-flash", {"input_tokens": 1}), + ("deepseek-flash", {"input_tokens": 1, "output_tokens": True}), + ("deepseek-flash", {"input_tokens": -1, "output_tokens": 1}), + ], +) +def test_estimated_cost_is_unknown_without_a_price_or_valid_counts(model, usage): + assert estimated_cost(model, usage) is None + + def test_generation_resolution_retries_until_cost_is_available(): responses = iter( [