From 9fce681f9e5a467b1bcf478a2c90c00ffc83b956 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Thu, 1 Oct 2026 20:29:02 +0800 Subject: [PATCH 1/6] feat(cost): estimate DeepSeek cost from token usage DeepSeek's official Anthropic-compatible endpoint returns token usage but no per-request cost, no generation id, and no usage or billing endpoint, so the existing OpenRouter usage/generation reconciliation cannot resolve a cost. Add estimated_cost(), which prices the returned input, output, and cache token counts with a static DeepSeek price list. It is used only after usage_cost() fails, so a provider-reported cost always wins, and it is marked kind: estimated to stay distinct from a reported or reconciled cost. --- src/nanopycodeagent/agent.py | 5 +++ src/nanopycodeagent/cost.py | 60 ++++++++++++++++++++++++++++++++++++ tests/test_cost.py | 47 ++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+) diff --git a/src/nanopycodeagent/agent.py b/src/nanopycodeagent/agent.py index cb4ee32..ced638e 100644 --- a/src/nanopycodeagent/agent.py +++ b/src/nanopycodeagent/agent.py @@ -42,6 +42,7 @@ from .atif import project_atif, write_atif from .bash_tool import run_bash from .cost import ( + estimated_cost, pending_cost, resolve_generation_cost, usage_cost, @@ -767,6 +768,10 @@ def _run_model_loop( ), "generation_id": generation_id, "cost": usage_cost(usage if isinstance(usage, dict) else None) + or estimated_cost( + str(getattr(message, "model", None) or model), + usage if isinstance(usage, dict) else None, + ) or pending_cost(generation_id), "duration_ms": (model_completed_ns - model_started_ns) / 1_000_000, "source_timestamp": utc_now(), diff --git a/src/nanopycodeagent/cost.py b/src/nanopycodeagent/cost.py index ea54ffb..a30d7eb 100644 --- a/src/nanopycodeagent/cost.py +++ b/src/nanopycodeagent/cost.py @@ -15,6 +15,66 @@ DEFAULT_RETRY_DELAYS = (1.0, 2.0, 4.0, 8.0, 15.0) RETRYABLE_HTTP_STATUSES = frozenset({404, 408, 409, 429, 500, 502, 503, 504}) +# Static price lists in USD per million tokens for providers that return token +# usage but not a per-request cost. DeepSeek's official endpoint is the current +# case: it reports ``input_tokens``, ``output_tokens``, and cache token counts, +# but no ``cost`` field and no generation id, so the existing usage and +# generation reconciliation paths cannot resolve a cost. Update these numbers +# when the provider changes its pricing. DeepSeek's cache-write price is zero. +_TOKEN_PRICES: dict[str, dict[str, Decimal]] = { + "deepseek-flash": { + "input": Decimal("0.3"), + "output": Decimal("1.2"), + "cache_read": Decimal("0.006"), + "cache_write": Decimal("0"), + }, +} +_TOKENS_PER_PRICE_UNIT = Decimal(1_000_000) + + +def estimated_cost(model: object, usage: JsonObject | None) -> JsonObject | None: + """Estimate a USD cost from token usage for a priced, non-reporting model. + + Returns ``None`` when the model has no local price list or the usage does + not carry the integer token counts the estimate needs. The result is marked + ``kind: estimated`` so it stays distinguishable from a provider-reported or + reconciled cost. Call this only after :func:`usage_cost` fails, so an + explicit provider cost always wins. + """ + if not isinstance(model, str) or not isinstance(usage, dict): + return None + prices = _TOKEN_PRICES.get(model) + if prices is None: + return None + if "input_tokens" not in usage or "output_tokens" not in usage: + return None + counts: dict[str, int] = {} + for field in ( + "input_tokens", + "output_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + ): + value = usage.get(field, 0) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return None + counts[field] = value + amount = ( + Decimal(counts["input_tokens"]) * prices["input"] + + Decimal(counts["cache_read_input_tokens"]) * prices["cache_read"] + + Decimal(counts["cache_creation_input_tokens"]) * prices["cache_write"] + + Decimal(counts["output_tokens"]) * prices["output"] + ) / _TOKENS_PER_PRICE_UNIT + if not amount.is_finite() or amount < 0: + return None + return { + "status": "resolved", + "amount": str(amount), + "currency": "USD", + "source": "token_estimate.deepseek", + "kind": "estimated", + } + def generation_url(base_url: object) -> str | None: """Build the provider-local generation endpoint from an SDK base URL.""" diff --git a/tests/test_cost.py b/tests/test_cost.py index d827378..9d7fb22 100644 --- a/tests/test_cost.py +++ b/tests/test_cost.py @@ -4,6 +4,7 @@ import pytest from nanopycodeagent.cost import ( + estimated_cost, generation_url, pending_cost, resolve_generation_cost, @@ -39,6 +40,52 @@ def test_usage_cost_preserves_provider_reported_decimal(): } +def test_estimated_cost_prices_deepseek_flash_token_usage(): + assert estimated_cost( + "deepseek-flash", + { + "input_tokens": 5719, + "output_tokens": 2203, + "cache_read_input_tokens": 25600, + "cache_creation_input_tokens": 0, + }, + ) == { + "status": "resolved", + "amount": "0.0045129", + "currency": "USD", + "source": "token_estimate.deepseek", + "kind": "estimated", + } + + +def test_estimated_cost_defaults_absent_cache_counts_to_zero(): + # A minimal DeepSeek-style usage object still prices without cache fields. + assert estimated_cost( + "deepseek-flash", {"input_tokens": 1000, "output_tokens": 1000} + ) == { + "status": "resolved", + "amount": "0.0015", + "currency": "USD", + "source": "token_estimate.deepseek", + "kind": "estimated", + } + + +@pytest.mark.parametrize( + ("model", "usage"), + [ + ("unknown-model", {"input_tokens": 1, "output_tokens": 1}), + ("deepseek-flash", None), + ("deepseek-flash", {}), + ("deepseek-flash", {"input_tokens": 1}), + ("deepseek-flash", {"input_tokens": 1, "output_tokens": True}), + ("deepseek-flash", {"input_tokens": -1, "output_tokens": 1}), + ], +) +def test_estimated_cost_is_unknown_without_a_price_or_valid_counts(model, usage): + assert estimated_cost(model, usage) is None + + def test_generation_resolution_retries_until_cost_is_available(): responses = iter( [ From f04913d9d46c16dff892d2a23adc2a694690fd8e Mon Sep 17 00:00:00 2001 From: minixalpha Date: Thu, 1 Oct 2026 20:29:02 +0800 Subject: [PATCH 2/6] fix(harbor): retry a transient 404 during agent setup Pinned task images occasionally resolve a package index that still points at removed .deb versions, so apt-get install fails with a 404 during agent setup before any model call. Retry the system-dependency transaction a bounded number of times; non-404 failures and the final attempt still propagate. --- .../harbor/src/harbor_adapter/adapter.py | 38 +++++++++- benchmarks/harbor/tests/test_adapter.py | 74 +++++++++++++++++++ 2 files changed, 108 insertions(+), 4 deletions(-) diff --git a/benchmarks/harbor/src/harbor_adapter/adapter.py b/benchmarks/harbor/src/harbor_adapter/adapter.py index 621eebb..94141ce 100644 --- a/benchmarks/harbor/src/harbor_adapter/adapter.py +++ b/benchmarks/harbor/src/harbor_adapter/adapter.py @@ -1,5 +1,6 @@ """Harbor adapter for running nanoPyCodeAgent in benchmark containers.""" +import asyncio import re import shlex import uuid @@ -8,6 +9,7 @@ from harbor.agents.installed.base import ( BaseInstalledAgent, CliFlag, + NonZeroAgentExitCodeError, with_prompt_template, ) from harbor.agents.model_connection import ModelConnectionSpec @@ -16,6 +18,11 @@ from harbor.models.trajectories.trajectory import Trajectory _DEFAULT_MAX_TURNS = 50 +_SYSTEM_DEPENDENCIES = ("curl", "bash", "git", "python3", "ca_certificates") +# Pinned task images occasionally resolve a package index that still points at +# removed `.deb` versions, producing a transient 404 during `apt-get install`. +# Retry the whole transaction a couple of times before failing setup. +_DEPENDENCY_RETRY_DELAYS = (2.0, 4.0) _PACKAGE_NAME = "nanoPyCodeAgent" _REPOSITORY_URL = "https://github.com/minixalpha/nanoPyCodeAgent.git" _UV_VERSION = "0.9.11" @@ -86,10 +93,7 @@ def _install_target(self) -> str: @override async def install(self, environment: BaseEnvironment) -> None: - await self.ensure_system_dependencies( - environment, - ("curl", "bash", "git", "python3", "ca_certificates"), - ) + await self._ensure_system_dependencies(environment) install_target = shlex.quote(self._install_target()) await self.exec_as_agent( environment, @@ -104,6 +108,32 @@ async def install(self, environment: BaseEnvironment) -> None: ), ) + async def _ensure_system_dependencies( + self, environment: BaseEnvironment + ) -> None: + """Install base dependencies, retrying a transient package-index 404. + + The retry only covers the system-package step, so it cannot replay any + model work; a non-404 failure or the final attempt propagates. + """ + total_attempts = len(_DEPENDENCY_RETRY_DELAYS) + 1 + for attempt, delay in enumerate((*_DEPENDENCY_RETRY_DELAYS, None)): + try: + await self.ensure_system_dependencies( + environment, _SYSTEM_DEPENDENCIES + ) + return + except NonZeroAgentExitCodeError as error: + if delay is None or "404" not in str(error): + raise + self.logger.warning( + "Retrying system dependency installation after HTTP 404 " + "(attempt %s/%s); no model call has started.", + attempt + 1, + total_attempts, + ) + await asyncio.sleep(delay) + def _runtime_env(self) -> dict[str, str]: model_connection = self.model_connection env: dict[str, str] = {} diff --git a/benchmarks/harbor/tests/test_adapter.py b/benchmarks/harbor/tests/test_adapter.py index a4c5a4a..3c59e18 100644 --- a/benchmarks/harbor/tests/test_adapter.py +++ b/benchmarks/harbor/tests/test_adapter.py @@ -6,6 +6,7 @@ from types import SimpleNamespace import pytest +from harbor.agents.installed.base import NonZeroAgentExitCodeError from harbor.models.agent.context import AgentContext from harbor_adapter import NanoPyCodeAgent @@ -103,6 +104,79 @@ def test_install_source_rejects_a_revision_coerced_to_a_number(tmp_path): make_adapter(tmp_path, git_ref=float("inf")) +def test_setup_retries_a_transient_404_and_then_succeeds(tmp_path, monkeypatch): + import harbor_adapter.adapter as adapter_module + + adapter = make_adapter(tmp_path) + attempts = [] + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + async def flaky(environment, dependencies): + attempts.append(dependencies) + if len(attempts) < 3: + raise NonZeroAgentExitCodeError( + "Command failed (exit 100): apt-get install -y curl\n" + "E: Failed to fetch http://deb.debian.org/.../curl.deb 404 Not Found" + ) + + monkeypatch.setattr(adapter, "ensure_system_dependencies", flaky) + monkeypatch.setattr(adapter_module.asyncio, "sleep", fake_sleep) + + asyncio.run(adapter._ensure_system_dependencies(RecordingEnvironment())) + + assert len(attempts) == 3 + assert sleeps == [2.0, 4.0] + + +def test_setup_does_not_retry_a_non_404_failure(tmp_path, monkeypatch): + import harbor_adapter.adapter as adapter_module + + adapter = make_adapter(tmp_path) + attempts = [] + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + async def failing(environment, dependencies): + attempts.append(dependencies) + raise NonZeroAgentExitCodeError("Command failed (exit 1): false") + + monkeypatch.setattr(adapter, "ensure_system_dependencies", failing) + monkeypatch.setattr(adapter_module.asyncio, "sleep", fake_sleep) + + with pytest.raises(NonZeroAgentExitCodeError): + asyncio.run(adapter._ensure_system_dependencies(RecordingEnvironment())) + + assert len(attempts) == 1 + assert sleeps == [] + + +def test_setup_gives_up_after_the_last_retry(tmp_path, monkeypatch): + import harbor_adapter.adapter as adapter_module + + adapter = make_adapter(tmp_path) + attempts = [] + + async def fake_sleep(delay): + pass + + async def always_404(environment, dependencies): + attempts.append(dependencies) + raise NonZeroAgentExitCodeError("E: 404 Not Found") + + monkeypatch.setattr(adapter, "ensure_system_dependencies", always_404) + monkeypatch.setattr(adapter_module.asyncio, "sleep", fake_sleep) + + with pytest.raises(NonZeroAgentExitCodeError): + asyncio.run(adapter._ensure_system_dependencies(RecordingEnvironment())) + + assert len(attempts) == 3 + + def test_run_pipes_the_instruction_and_forwards_anthropic_configuration(tmp_path): instruction = 'fix "quoted" input; echo $TOKEN\nthen run the tests' adapter = make_adapter( From ff6f6e48c3ac7f507e95db8df56e292f0ec57da1 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Thu, 1 Oct 2026 20:29:02 +0800 Subject: [PATCH 3/6] docs(benchmarks): record the DeepSeek official pilot20 Add the structured result and report for running the 20-task pilot set on the DeepSeek official endpoint with deepseek-flash (16/18 scored, two infrastructure exceptions, token-estimated cost), plus the changelog entries for the cost estimator and the setup retry. --- .../reports/deepseek-official-20260930.md | 86 ++++ ...21-deepseek-official-pilot20-20260930.json | 387 ++++++++++++++++++ docs/changelogs/0.8.x.md | 11 + 3 files changed, 484 insertions(+) create mode 100644 benchmarks/harbor/reports/deepseek-official-20260930.md create mode 100644 benchmarks/harbor/results/tb21-deepseek-official-pilot20-20260930.json diff --git a/benchmarks/harbor/reports/deepseek-official-20260930.md b/benchmarks/harbor/reports/deepseek-official-20260930.md new file mode 100644 index 0000000..4b2af9c --- /dev/null +++ b/benchmarks/harbor/reports/deepseek-official-20260930.md @@ -0,0 +1,86 @@ +# DeepSeek official endpoint pilot20 + +Switched the 20-task pilot set from OpenRouter to DeepSeek's official +Anthropic-compatible endpoint and re-ran it with `deepseek-flash`. The run +scored **16/18 scored trials** (Harbor mean 0.80); two trials were lost to +infrastructure failures rather than model failures. + +The complete configuration, per-trial usage, token-estimated cost, and balance +snapshots are in the +[public results](../results/tb21-deepseek-official-pilot20-20260930.json). Raw +evidence remains in the recorded Git-ignored `jobs/` directories. + +## Configuration + +| Setting | Value | +| --- | --- | +| Job | `tb21-deepseek-official-pilot20-20260930` | +| Agent | `e75cacb6` (`0.8.1.dev45`), locally built and verified wheel | +| Endpoint | `https://api.deepseek.com/anthropic` (Anthropic Messages) | +| Model | `deepseek-flash` | +| Tasks | the pinned 20-task pilot set | +| Budget | max_turns 100, max_tokens 65536, per-task agent budget native−180 s | +| Concurrency / attempts / retries | 2 / 1 / 0 | +| Provider pin / HTTP recorder | none — the provider is the only change | + +## Result + +| Bucket | Count | +| --- | ---: | +| Planned trials | 20 | +| Scored trials | 18 | +| Passed | 16 | +| Reward 0 | 2 | +| Infrastructure exceptions | 2 | +| Pass rate over scored | 88.9% | + +- Reward 0: `torch-tensor-parallelism`, `pytorch-model-recovery`. +- `qemu-alpine-ssh`: agent setup failed fetching pinned Debian packages with + HTTP 404; no model call happened. +- `torch-pipeline-parallelism`: the agent completed, but the official verifier + exceeded its native 900 s limit. + +Token usage: 30,933,108 prompt (30,586,240 cached, 346,868 uncached), 891,020 +completion. + +## Cost without a provider cost API + +DeepSeek's official API reports token usage but no per-request cost, no +generation id, and no usage/billing endpoint — `usage.cost` is absent, +`x-generation-id` is absent, and `/v1/generation`, `/user/usage`, +`/billing/usage`, and `/pricing` all return 404. The previous OpenRouter +reconciliation path therefore cannot resolve a DeepSeek cost. + +This branch adds a **token-based estimator** (see Implementation) that prices +the returned tokens with a static DeepSeek price list (USD per million: +input 0.3, cache read 0.006, cache write 0, output 1.2). It marks the result +`kind: estimated`, so it stays distinguishable from provider-reported cost. + +| Component | Tokens | Rate | Cost (USD) | +| --- | ---: | ---: | ---: | +| Uncached input | 346,868 | 0.3 | 0.104060 | +| Cache read | 30,586,240 | 0.006 | 0.183517 | +| Output | 891,020 | 1.2 | 1.069224 | +| **Total** | | | **1.356802** | + +An account balance delta (`/user/balance`, ¥106.63 → ¥102.01) is recorded as an +upper bound only: the interactive assistant session used the same account +during the run, so it is not an isolated run cost, and it is denominated in CNY. + +## Implementation + +- `cost.py`: add `estimated_cost(model, usage)` and a static DeepSeek price + table; `agent.py` uses it only after `usage_cost` fails, so an explicit + provider cost always wins. +- `harbor_adapter`: retry the system-dependency `apt-get` transaction once or + twice on a transient HTTP 404 before failing setup. In the follow-up qemu + re-run the retry fired on attempts 1/3 and 2/3, but the pinned + `bullseye-security` index references `.deb` versions that are gone, so the 404 + is deterministic and a retry alone cannot fix that task. The historical fix + preloads a verified `.deb` cache from the original APT index. + +## Comparison limits + +The OpenRouter pilot20 runs used provider pinning, a different snapshot date, +and different instrumentation, and this run has two infrastructure exceptions. +This is a provider-switch record, not a controlled causal comparison. diff --git a/benchmarks/harbor/results/tb21-deepseek-official-pilot20-20260930.json b/benchmarks/harbor/results/tb21-deepseek-official-pilot20-20260930.json new file mode 100644 index 0000000..ebd66c2 --- /dev/null +++ b/benchmarks/harbor/results/tb21-deepseek-official-pilot20-20260930.json @@ -0,0 +1,387 @@ +{ + "schema_version": 1, + "generated_at": "2026-10-01T12:28:08.216679+00:00", + "job_name": "tb21-deepseek-official-pilot20-20260930", + "run_date": "2026-09-30", + "scope": "Single-attempt provider switch of the 20-task pilot set from OpenRouter to the DeepSeek official Anthropic-compatible endpoint; not a full Terminal-Bench 2.1 score or a controlled causal comparison.", + "experiment": { + "agent_ref": "e75cacb6a780314471ff12655ecb6fe414109ee4", + "wheel_sha256": "2be853f2dcbaef286295aacfbc2c074455d41b8531d7d585fdc727eeecaa9d73", + "harbor_version": "0.21.0", + "dataset": "terminal-bench/terminal-bench-2-1", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "model": "deepseek/deepseek-flash", + "endpoint": "https://api.deepseek.com/anthropic", + "api": "Anthropic Messages", + "model_settings_source": "~/.nanoPyCodeAgent/settings.json", + "limits": { + "n_tasks": 20, + "n_attempts": 1, + "n_concurrent": 2, + "max_turns": 100, + "max_tokens_per_response": 65536, + "time_budget": "native task timeout - 180s, computed per task", + "harbor_max_retries": 0, + "harbor_timeout_overrides": false, + "openrouter_provider_pin": null, + "http_recorder": null, + "temperature": "not explicitly set", + "reasoning_effort": "not explicitly set" + } + }, + "scores": { + "planned_trials": 20, + "scored_trials": 18, + "passed_trials": 16, + "failed_reward_trials": 2, + "unscored_trials": 2, + "exception_counts": { + "VerifierTimeoutError": 1, + "NonZeroAgentExitCodeError": 1 + }, + "harbor_eval_mean": 0.8, + "pass_rate_over_planned": 0.8, + "pass_rate_over_scored": 0.8888888888888888 + }, + "usage": { + "total_prompt_tokens": 30933108, + "total_cached_tokens": 30586240, + "total_cache_write_tokens": 0, + "total_uncached_input_tokens": 346868, + "total_completion_tokens": 891020, + "complete": true + }, + "cost": { + "currency": "USD", + "method": "token_estimate.deepseek", + "source": "static DeepSeek official price table", + "prices_per_million_tokens": { + "input": "0.3", + "output": "1.2", + "cache_read": "0.006", + "cache_write": "0" + }, + "token_estimated_total_usd": "1.35680184", + "reported_by_provider": false, + "note": "DeepSeek's official API returns token usage but no per-request cost, no generation id, and no usage/billing endpoint. This total is computed from token usage, not reported by the provider.", + "account_balance_snapshots_cny": [ + { + "label": "before", + "at": "2026-09-30T14:05:18.075037+00:00", + "is_available": true, + "balance_infos": [ + { + "currency": "CNY", + "total_balance": "106.63", + "granted_balance": "0.00", + "topped_up_balance": "106.63" + } + ] + }, + { + "label": "after", + "at": "2026-09-30T16:14:44.312896+00:00", + "is_available": true, + "balance_infos": [ + { + "currency": "CNY", + "total_balance": "102.01", + "granted_balance": "0.00", + "topped_up_balance": "102.01" + } + ] + } + ], + "account_balance_delta_cny": "4.62", + "account_balance_caveat": "The interactive assistant session used the same DeepSeek account during the run, so the balance delta is an upper bound, not an isolated run cost." + }, + "trials": [ + { + "task": "terminal-bench/caffe-cifar-10", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 3600, + "usage": { + "prompt_tokens": 932801, + "cache_read_tokens": 914048, + "cache_write_tokens": 0, + "uncached_input_tokens": 18753, + "completion_tokens": 25539 + }, + "estimated_cost_usd": "0.041756988" + }, + { + "task": "terminal-bench/circuit-fibsqrt", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 3600, + "usage": { + "prompt_tokens": 972406, + "cache_read_tokens": 963328, + "cache_write_tokens": 0, + "uncached_input_tokens": 9078, + "completion_tokens": 64174 + }, + "estimated_cost_usd": "0.085512168" + }, + { + "task": "terminal-bench/dna-assembly", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 1800, + "usage": { + "prompt_tokens": 2356256, + "cache_read_tokens": 2331008, + "cache_write_tokens": 0, + "uncached_input_tokens": 25248, + "completion_tokens": 88429 + }, + "estimated_cost_usd": "0.127675248" + }, + { + "task": "terminal-bench/kv-store-grpc", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 32653, + "cache_read_tokens": 29568, + "cache_write_tokens": 0, + "uncached_input_tokens": 3085, + "completion_tokens": 2825 + }, + "estimated_cost_usd": "0.004492908" + }, + { + "task": "terminal-bench/llm-inference-batching-scheduler", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 1800, + "usage": { + "prompt_tokens": 1079376, + "cache_read_tokens": 1061248, + "cache_write_tokens": 0, + "uncached_input_tokens": 18128, + "completion_tokens": 52222 + }, + "estimated_cost_usd": "0.074472288" + }, + { + "task": "terminal-bench/log-summary-date-ranges", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 27223, + "cache_read_tokens": 22656, + "cache_write_tokens": 0, + "uncached_input_tokens": 4567, + "completion_tokens": 1777 + }, + "estimated_cost_usd": "0.003638436" + }, + { + "task": "terminal-bench/merge-diff-arc-agi-task", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 274857, + "cache_read_tokens": 260224, + "cache_write_tokens": 0, + "uncached_input_tokens": 14633, + "completion_tokens": 16171 + }, + "estimated_cost_usd": "0.025356444" + }, + { + "task": "terminal-bench/model-extraction-relu-logits", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 538556, + "cache_read_tokens": 526592, + "cache_write_tokens": 0, + "uncached_input_tokens": 11964, + "completion_tokens": 29094 + }, + "estimated_cost_usd": "0.041661552" + }, + { + "task": "terminal-bench/mteb-leaderboard", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 3600, + "usage": { + "prompt_tokens": 1719176, + "cache_read_tokens": 1680128, + "cache_write_tokens": 0, + "uncached_input_tokens": 39048, + "completion_tokens": 29412 + }, + "estimated_cost_usd": "0.057089568" + }, + { + "task": "terminal-bench/openssl-selfsigned-cert", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 40143, + "cache_read_tokens": 37120, + "cache_write_tokens": 0, + "uncached_input_tokens": 3023, + "completion_tokens": 3902 + }, + "estimated_cost_usd": "0.00581202" + }, + { + "task": "terminal-bench/path-tracing", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 1800, + "usage": { + "prompt_tokens": 1140749, + "cache_read_tokens": 1084672, + "cache_write_tokens": 0, + "uncached_input_tokens": 56077, + "completion_tokens": 35038 + }, + "estimated_cost_usd": "0.065376732" + }, + { + "task": "terminal-bench/pypi-server", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 110863, + "cache_read_tokens": 105216, + "cache_write_tokens": 0, + "uncached_input_tokens": 5647, + "completion_tokens": 5713 + }, + "estimated_cost_usd": "0.009180996" + }, + { + "task": "terminal-bench/pytorch-model-recovery", + "reward": 0.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 1481698, + "cache_read_tokens": 1460352, + "cache_write_tokens": 0, + "uncached_input_tokens": 21346, + "completion_tokens": 44263 + }, + "estimated_cost_usd": "0.068281512" + }, + { + "task": "terminal-bench/qemu-alpine-ssh", + "reward": null, + "exception_type": "NonZeroAgentExitCodeError", + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "uncached_input_tokens": 0, + "completion_tokens": 0 + }, + "estimated_cost_usd": "0.000" + }, + { + "task": "terminal-bench/regex-chess", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 3600, + "usage": { + "prompt_tokens": 6867208, + "cache_read_tokens": 6828288, + "cache_write_tokens": 0, + "uncached_input_tokens": 38920, + "completion_tokens": 122473 + }, + "estimated_cost_usd": "0.199613328" + }, + { + "task": "terminal-bench/regex-log", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 435322, + "cache_read_tokens": 429824, + "cache_write_tokens": 0, + "uncached_input_tokens": 5498, + "completion_tokens": 62276 + }, + "estimated_cost_usd": "0.078959544" + }, + { + "task": "terminal-bench/schemelike-metacircular-eval", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 2400, + "usage": { + "prompt_tokens": 11204578, + "cache_read_tokens": 11154688, + "cache_write_tokens": 0, + "uncached_input_tokens": 49890, + "completion_tokens": 192178 + }, + "estimated_cost_usd": "0.312508728" + }, + { + "task": "terminal-bench/torch-pipeline-parallelism", + "reward": null, + "exception_type": "VerifierTimeoutError", + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 1334231, + "cache_read_tokens": 1318656, + "cache_write_tokens": 0, + "uncached_input_tokens": 15575, + "completion_tokens": 62393 + }, + "estimated_cost_usd": "0.087456036" + }, + { + "task": "terminal-bench/torch-tensor-parallelism", + "reward": 0.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 168404, + "cache_read_tokens": 164992, + "cache_write_tokens": 0, + "uncached_input_tokens": 3412, + "completion_tokens": 18918 + }, + "estimated_cost_usd": "0.024715152" + }, + { + "task": "terminal-bench/write-compressor", + "reward": 1.0, + "exception_type": null, + "native_agent_timeout_seconds": 900, + "usage": { + "prompt_tokens": 216608, + "cache_read_tokens": 213632, + "cache_write_tokens": 0, + "uncached_input_tokens": 2976, + "completion_tokens": 34223 + }, + "estimated_cost_usd": "0.043242192" + } + ], + "qemu_alpine_ssh_retry": { + "job": "tb21-deepseek-official-qemu-retry-20260930", + "exception_type": "NonZeroAgentExitCodeError", + "retry": "The tracked 404 retry fired on attempts 1/3 and 2/3 and still failed.", + "finding": "The pinned Debian bullseye-security index references .deb versions that are no longer in the pool; the 404 is deterministic, so a retry alone cannot fix this task. The historical fix preloads a verified .deb cache from the original APT index." + }, + "comparison_limits": "The OpenRouter pilot20 runs used provider pinning, a different snapshot date, and different instrumentation. Two exceptions here are infrastructure failures (apt 404 and a verifier timeout), not model failures." +} diff --git a/docs/changelogs/0.8.x.md b/docs/changelogs/0.8.x.md index 8107711..125ecaf 100644 --- a/docs/changelogs/0.8.x.md +++ b/docs/changelogs/0.8.x.md @@ -5,6 +5,13 @@ All notable changes in the **0.8.x** release series are documented here. ## [Unreleased] ### Added +- Estimate model-call costs from token usage when a provider reports tokens + but no per-request cost, starting with the DeepSeek official + Anthropic-compatible endpoint. The estimate uses a static DeepSeek price + list, is marked `kind: estimated` so it stays distinct from a + provider-reported or reconciled cost, and is only used after a real + provider-reported cost is absent. DeepSeek exposes no cost or generation + endpoint, so the existing OpenRouter reconciliation cannot resolve it. - Configure per-reply generation budgets through `--max-tokens` or `ANTHROPIC_MAX_TOKENS`, including the user settings file and Harbor adapter. Record the effective limit in startup output, Event Journals, and ATIF @@ -25,6 +32,10 @@ All notable changes in the **0.8.x** release series are documented here. supported. ### Fixed +- Retry the Harbor adapter's system-dependency installation a bounded number + of times when the package index returns a transient HTTP 404, instead of + failing agent setup immediately. Non-404 failures and the final attempt still + propagate. - Terminate a timed-out or interrupted bash command's process group on POSIX and other members of its session on Linux, including GNU `timeout` children in separate groups. Close output pipes without draining detached children From 6e4cc23deae34ef1d82af96c4679f06bd4db9ae9 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Sun, 4 Oct 2026 21:10:44 +0800 Subject: [PATCH 4/6] feat(harbor): standardize benchmark setup and verifier retries Preserve estimated-cost metadata in ATIF and Harbor aggregates, and allow generation billing reconciliation to replace token-based estimates. Add a recorded working-tree runner, SHA256-verified APT cache, and a pinned QEMU dependency preflight. Retry only verifier timeouts, up to three native attempts, while preserving logs and keeping setup errors separate from scores. Document the workflow in both languages and cover its contracts in CI. Validation: 345 core tests passed with 4 skipped; 74 Harbor tests passed. The full QEMU trial passed its official verifier on the first attempt. --- .github/workflows/ci.yml | 16 ++ AGENTS.md | 18 ++ benchmarks/harbor/README.md | 117 +++++++- benchmarks/harbor/README.zh-CN.md | 88 +++++- .../harbor/configs/qemu-alpine-ssh.json | 21 ++ .../harbor/src/harbor_adapter/__init__.py | 3 +- .../harbor/src/harbor_adapter/__main__.py | 182 ++++++++++++ .../src/harbor_adapter/_verifier_runner.py | 143 ++++++++++ .../harbor/src/harbor_adapter/adapter.py | 106 ++++++- .../harbor/src/harbor_adapter/bootstrap.py | 179 ++++++++++++ .../harbor/src/harbor_adapter/verifier.py | 139 ++++++++++ .../harbor/tests/test_atif_compatibility.py | 52 ++++ benchmarks/harbor/tests/test_bootstrap.py | 186 +++++++++++++ benchmarks/harbor/tests/test_verifier.py | 259 ++++++++++++++++++ benchmarks/harbor/tests/test_workflow.py | 75 +++++ docs/changelogs/0.8.x.md | 10 + src/nanopycodeagent/agent.py | 4 +- src/nanopycodeagent/atif.py | 12 + tests/test_agent_events.py | 69 +++++ tests/test_atif.py | 43 +++ 20 files changed, 1706 insertions(+), 16 deletions(-) create mode 100644 benchmarks/harbor/configs/qemu-alpine-ssh.json create mode 100644 benchmarks/harbor/src/harbor_adapter/__main__.py create mode 100644 benchmarks/harbor/src/harbor_adapter/_verifier_runner.py create mode 100644 benchmarks/harbor/src/harbor_adapter/bootstrap.py create mode 100644 benchmarks/harbor/src/harbor_adapter/verifier.py create mode 100644 benchmarks/harbor/tests/test_bootstrap.py create mode 100644 benchmarks/harbor/tests/test_verifier.py create mode 100644 benchmarks/harbor/tests/test_workflow.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f96079..69cc8d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,3 +41,19 @@ jobs: run: uv run --frozen pytest - name: Test the httpx2 SDK transport run: uv run --with anthropic==1.5.0 pytest + + harbor: + name: Harbor workflow contracts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + python-version: "3.13" + - name: Test cache integrity, preflight, and verifier retries + run: >- + uv run --frozen --project benchmarks/harbor pytest + -c benchmarks/harbor/pyproject.toml benchmarks/harbor/tests diff --git a/AGENTS.md b/AGENTS.md index 492f57d..ce1cd67 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,6 +19,24 @@ the matching one: See `docs/RELEASING.md` for the release process and prerequisites. +## Benchmarks + +For new Harbor benchmark runs, use the standard workflow from the repository +root: `uv run --project benchmarks/harbor python -m harbor_adapter run --config +`. See `benchmarks/harbor/README.md` for configuration, verified APT +cache import, and the `--install-only` dependency preflight option. + +- Do not create per-job adapter copies or ad hoc bootstrap scripts. Add reusable + fixes and reviewed task-revision profiles to `benchmarks/harbor` with tests. +- Preserve task images, package sources, and official test scripts. Cache hits + must match the current package index's filename, size, and SHA256 digest. +- Run dependency preflight before model work when a task has a registered + profile. Record unsupported profiles explicitly; do not claim they passed. +- Keep setup failures and verifier errors separate from scored failures. Only + verifier timeouts may retry, at most three total attempts using the same agent + output and the task's original per-attempt time limit. + + ## Language Write everything in **English** by default — source code (identifiers, diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index fdd41cd..3b841aa 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -18,8 +18,81 @@ export ANTHROPIC_API_KEY="..." export ANTHROPIC_BASE_URL="https://gateway.example" ``` -From the repository root, run Harbor through this workspace and pin the agent -installed in the task container: +Run new benchmarks through the standard entry point, from the repository root: + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter run \ + --config benchmarks/harbor/configs/qemu-alpine-ssh.json +``` + +This example pins the QEMU task revision and uses `deepseek/deepseek-flash`. +Set the endpoint and credentials for that model, or copy the configuration and +change `agents[].model_name`, task selection, and budgets for another run. These +are ordinary Harbor JSON configurations; the entry point supplies the standard +verifier and cache policy. It accepts the repository's `NanoPyCodeAgent` adapter +with budget kwargs; credentials belong in the environment. + +The entry point builds the current working tree, including uncommitted changes, +into a wheel and checks its Python sources against the recorded snapshot. It +also snapshots the adapter, records source and wheel hashes, and uses that +snapshot throughout the run. Inputs and logs live in `jobs/-input/`; +Harbor outputs remain in `jobs//`. Existing jobs are never overwritten. +`summary.json` records scores, setup/preflight status, cache use, and verifier +attempts. An infrastructure exception has no score; it is not converted to zero. +The command exits nonzero for execution errors or incomplete results, while a +completed score of zero is a valid benchmark result. + +Use `--job-name ` for a readable run name. Add `--prepare-only` to +build and record inputs without starting containers. Add `--install-only` to +exercise dependency installation and preflight without calling the model or +executing the official tests. A full run still performs setup and preflight in +its own fresh container; an install-only check does not resume into model work. + +### Verified dependencies and preflight + +The standard adapter automatically uses `.cache/nanopy-harbor/`, shared across +runs and ignored by Git. Override it with `NANOPY_HARBOR_CACHE` or the entry +point's `--cache-dir`. On APT systems it refreshes the package index, resolves a +SHA256 download plan, restores matching cached files, and downloads missing +ones. Successful downloads are verified and saved before installation, so +APT cleanup hooks cannot erase the shared copies. A different version or digest +does not reuse an old cached file. Corrupt files fail setup before model work. +Other package managers use Harbor's regular dependency installation. + +To import previously verified downloads once: + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter import-apt \ + /path/to/bootstrap-cache/manifest.json +``` + +The manifest is a JSON array of objects with `url`, `filename`, `size`, and +`sha256`, accompanied by `packages/` files. Import validates every +file. At use time, the current APT index must independently match each entry. +Copying the entire shared cache to another machine is also supported. A new +machine with no matching cache still needs the original downloads to be +available; this mechanism does not replace package sources or invent missing +versions. + +Verifier dependency profiles are tied to reviewed task revisions in +`src/harbor_adapter/bootstrap.py`. The current QEMU profile installs `sshpass` +and warms the original verifier's uv 0.9.5, Python 3.13, pytest 8.4.1, and +pytest-json-ctrf 0.3.5 dependencies using `pytest --version`. It runs no task +assertions or solution code. Unknown QEMU revisions fail setup pending profile +review; tasks without a profile explicitly report `not_configured`. Add a +profile and regression tests when another task needs verifier preflight. + +Every installation writes `agent/bootstrap.json`, including partial progress +on failure. Preflight failure stops the trial before any model call. The +official verifier script remains unchanged and may still access the network +when it runs; a successful preflight is not a guarantee against later outages. +Full standard runs enable the timeout retry policy below by default, with zero +whole-trial retries. CI checks these contracts without model calls or Docker. + +### Direct Harbor invocation + +For adapter development or comparison with published versions, the lower-level +Harbor interface remains available. Pin the agent installed in the container: ```bash uv run --project benchmarks/harbor harbor run \ @@ -27,6 +100,8 @@ uv run --project benchmarks/harbor harbor run \ --agent harbor_adapter:NanoPyCodeAgent \ --agent-kwarg git_ref= \ --model anthropic/claude-sonnet-4-6 \ + --verifier harbor_adapter:RetryingVerifier \ + --verifier-timeout-multiplier 4 \ --env docker \ --n-concurrent 1 \ --n-attempts 1 @@ -65,6 +140,11 @@ The adapter also asks the agent to write an ATIF-v1.7 trajectory directly to ATIF output and backfills prompt, completion, cache-token, and cost totals into the agent result. Missing, invalid, or partial trajectories remain explicitly diagnosed; unknown usage or cost is not reported as zero. +If any costs are estimated, `final_metrics.extra` records +`cost_is_estimated: true` and the estimated portion in `estimated_cost_usd`. +The adapter preserves both fields in the agent result's `metadata.trajectory`, +including for partial totals. Estimates with a generation ID remain eligible +for billing reconciliation; a successful lookup replaces the estimate. By default, the adapter strips the first provider prefix from `--model` and passes the result to nanoPyCodeAgent as `ANTHROPIC_MODEL`. Set @@ -74,6 +154,39 @@ precedence. Harbor-native provider credentials and configured base URLs are also normalized to the `ANTHROPIC_*` variables expected by nanoPyCodeAgent's SDK. +## Retry verifier timeouts + +The example above uses `harbor_adapter:RetryingVerifier` for single-step Linux +tasks. It runs the original test script up to **three times total**, retrying +only when the script exceeds the task's native `[verifier].timeout_sec`. A +900-second task therefore gets at most three 900-second attempts. A completed +verification stops immediately, including a reward of zero. Script errors, +missing or invalid rewards, and cancellation do not trigger another attempt. + +`--verifier-timeout-multiplier 4` gives Harbor's outer watchdog room for all +three attempts plus test upload, process cleanup, and log collection. It does +not extend each test-script attempt. Conflicting timeout overrides or caps +that leave insufficient overall time are rejected. Use +`--verifier-kwarg max_attempts=2` for two total attempts, or `max_attempts=1` for +one; values above three are rejected. + +The runner reuses the same container, agent output, and dependency caches; +the agent is run once. Test scripts must tolerate being rerun in that +environment; verifier-side filesystem changes are not rolled back. Timed-out +process groups and other members of their Linux session are killed before +retrying. Each attempt's logs and any reward files are archived under +`verifier/attempts/1/`, `2/`, and `3/`. The runner clears current verifier +outputs before the next attempt, so a stale reward cannot determine its score. +`verifier/retry-summary.json` records each attempt's status, duration, and exit +code when available. Exhausting the attempts still produces Harbor's +`VerifierTimeoutError`. + +This verifier requires `python3` and `bash` in the container, which the +nanoPyCodeAgent adapter installs. These options apply to new runs; they cannot +restore a deleted container or missing task outputs from an old trial. +Harbor's ordinary `--max-retries` setting reruns the entire trial, including +the agent, and is independent of these verifier attempts. + ## Test the adapter ```bash diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index 3b84682..12ca806 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -17,7 +17,63 @@ export ANTHROPIC_API_KEY="..." export ANTHROPIC_BASE_URL="https://gateway.example" ``` -从仓库根目录通过这个 workspace 运行 Harbor，并固定安装到 task 容器中的 agent： +新的 benchmark 统一从仓库根目录通过标准入口运行： + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter run \ + --config benchmarks/harbor/configs/qemu-alpine-ssh.json +``` + +此示例固定了 QEMU 题目版本，模型为 `deepseek/deepseek-flash`。请配置对应的 +endpoint 和凭证；测试其他模型或题目时，复制配置并修改 `agents[].model_name`、 +题目选择及预算。这是普通的 Harbor JSON 配置，入口会补齐标准验证器和缓存策略。 +入口接受本仓库的 `NanoPyCodeAgent` adapter 和预算参数，凭证通过环境变量传入。 + +入口会将当前工作区（包括未提交修改）构建为 wheel，并核对其 Python 源码与保存的 +快照一致；同时保存 adapter 快照及源码、wheel 的哈希，整个运行都使用该快照。 +输入和日志保存在 `jobs/-input/`，Harbor 输出保存在 +`jobs//`，不会覆盖已有运行。`summary.json` 记录评分、安装和预检状态、 +缓存使用情况及验证次数。基础设施异常保留为无评分，不会转换为 0 分。 +执行错误或结果不完整时命令返回非零退出码；正常完成验证得到 0 分仍是有效结果。 + +可用 `--job-name ` 命名运行。`--prepare-only` 只构建并保存输入， +不启动容器；`--install-only` 只执行依赖安装和预检，不调用模型、不运行官方测试。 +完整运行仍会在自己的新容器里执行安装和预检，不能把 install-only 容器续跑为解题。 + +### 依赖缓存与预检 + +标准 adapter 自动使用被 Git 忽略的 `.cache/nanopy-harbor/` 跨运行共享缓存， +可通过 `NANOPY_HARBOR_CACHE` 或入口的 `--cache-dir` 覆盖。对于 APT 系统，先更新 +索引并解析 SHA256 下载计划，再恢复匹配的缓存文件、下载缺失文件。下载成功后会先 +校验并保存缓存，再安装，避免 APT 清理钩子删除共享副本。版本或哈希不匹配时不会 +复用旧包，文件损坏会在模型调用前终止安装。其他包管理器使用 Harbor 的常规安装。 + +历史下载只需导入一次： + +```bash +uv run --project benchmarks/harbor python -m harbor_adapter import-apt \ + /path/to/bootstrap-cache/manifest.json +``` + +manifest 是包含 `url`、`filename`、`size`、`sha256` 的 JSON 对象数组，同目录下 +配套 `packages/` 文件。导入时逐个校验，使用时还必须独立匹配当前 APT +索引。也可将整个共享缓存复制到其他机器。新机器没有匹配缓存时仍依赖原下载地址 +可用；此机制不会更换软件源或伪造缺失版本。 + +验证器依赖预检配置位于 `src/harbor_adapter/bootstrap.py`，绑定已检查的题目版本。 +目前 QEMU 配置会安装 `sshpass`，并通过 `pytest --version` 预热原验证器所需的 +uv 0.9.5、Python 3.13、pytest 8.4.1 和 pytest-json-ctrf 0.3.5，不执行题目断言或 +参考解。未知的 QEMU 版本会在安装阶段报错，要求先检查对应预检配置；没有预检配置 +的其他题目明确记录为 `not_configured`。其他题目需要预检时，在这里新增配置及回归测试。 + +每次安装都会写入 `agent/bootstrap.json`，失败时也保留已完成的步骤。预检失败会 +在模型调用前终止 trial。官方验证脚本保持原样，执行时仍可能访问网络，所以预检 +通过不能保证后续没有网络故障。完整标准运行默认启用下述验证超时重试，整题重试 +次数为零。CI 会验证这些约定，不调用模型、不启动 Docker。 + +### 直接调用 Harbor + +开发 adapter 或对照已发布版本时，仍可使用底层 Harbor 入口，并固定容器内 agent 版本： ```bash uv run --project benchmarks/harbor harbor run \ @@ -25,6 +81,8 @@ uv run --project benchmarks/harbor harbor run \ --agent harbor_adapter:NanoPyCodeAgent \ --agent-kwarg git_ref= \ --model anthropic/claude-sonnet-4-6 \ + --verifier harbor_adapter:RetryingVerifier \ + --verifier-timeout-multiplier 4 \ --env docker \ --n-concurrent 1 \ --n-attempts 1 @@ -56,6 +114,10 @@ adapter 还会要求 agent 将 ATIF-v1.7 trajectory 直接写入 `/logs/agent/trajectory.json`。Harbor 会把该文件作为 trial 的原生 ATIF 输出采集， 并将 prompt、completion、cache token 和 cost 汇总回填到 agent result。trajectory 缺失、无效或指标不完整时会保留明确诊断；未知 usage 或 cost 不会被记成零。 +只要费用中包含估算，`final_metrics.extra` 就会记录 `cost_is_estimated: true`， +并通过 `estimated_cost_usd` 单独记录估算部分的金额。adapter 会将这两个字段保留到 +agent result 的 `metadata.trajectory`，费用仅部分已知时也一样。带 generation ID +的估算费用仍会进行账单补查；补查成功后，真实费用会替换估算值。 默认情况下，adapter 会移除 `--model` 中的第一个 provider 前缀，并将结果作为 `ANTHROPIC_MODEL` 传给 nanoPyCodeAgent。只有自定义 endpoint 要求的实际模型名与 @@ -63,6 +125,30 @@ Harbor 的 `provider/model` 身份不同时，才显式设置 `ANTHROPIC_MODEL` Harbor 原生的 provider 凭证和已经配置的 base URL 也会转换为 nanoPyCodeAgent SDK 所需的 `ANTHROPIC_*` 变量。 +## 验证器超时重试 + +上面的示例使用 `harbor_adapter:RetryingVerifier`，适用于单阶段 Linux 题目。 +原始测试脚本**总共最多执行 3 次**，仅当脚本超过题目原生的 +`[verifier].timeout_sec` 时才重试。900 秒时限的题目，每次仍最多执行 900 秒。 +验证完成后立即停止，包括得分为 0 的情况。脚本错误、评分缺失或格式错误、取消执行 +都不会触发下一次尝试。 + +`--verifier-timeout-multiplier 4` 为 Harbor 外层时限预留 3 次尝试以及上传测试、 +清理进程和收集日志的时间，不会延长每次测试脚本的时限。如果其他超时覆盖或上限 +导致总时间不足，会报错。可用 `--verifier-kwarg max_attempts=2` 将总尝试次数设为 +2 次，或用 `max_attempts=1` 只执行一次；不允许设置为超过 3 次。 + +重试复用同一容器、agent 产物和依赖缓存，agent 只运行一次。测试脚本需要能够在该 +环境中重复执行，验证过程对文件系统的修改不会回滚。超时后会先终止测试进程组以及 +同一 Linux session 中的其他进程，再开始下一次尝试。每次日志及评分文件归档到 +`verifier/attempts/1/`、`2/`、`3/`。下一次尝试开始前会清理当前验证输出，避免旧 +评分被误用。`verifier/retry-summary.json` 记录每次状态、耗时及可用的退出码。 +所有尝试均超时后，仍以 Harbor 的 `VerifierTimeoutError` 结束。 + +容器需要 `python3` 和 `bash`，nanoPyCodeAgent adapter 会安装它们。这些选项用于 +新运行，无法恢复旧 trial 已删除的容器或缺失的任务产物。Harbor 原生的 +`--max-retries` 会重新执行整道题（包括 agent），与这里的验证尝试次数相互独立。 + ## 测试 adapter ```bash diff --git a/benchmarks/harbor/configs/qemu-alpine-ssh.json b/benchmarks/harbor/configs/qemu-alpine-ssh.json new file mode 100644 index 0000000..e33e613 --- /dev/null +++ b/benchmarks/harbor/configs/qemu-alpine-ssh.json @@ -0,0 +1,21 @@ +{ + "n_concurrent_trials": 1, + "n_attempts": 1, + "agents": [ + { + "name": "harbor_adapter:NanoPyCodeAgent", + "model_name": "deepseek/deepseek-flash", + "kwargs": { + "max_turns": 100, + "max_tokens": 65536, + "time_budget_seconds": 720 + } + } + ], + "tasks": [ + { + "name": "terminal-bench/qemu-alpine-ssh", + "ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28" + } + ] +} diff --git a/benchmarks/harbor/src/harbor_adapter/__init__.py b/benchmarks/harbor/src/harbor_adapter/__init__.py index 915d6fd..5e8c83c 100644 --- a/benchmarks/harbor/src/harbor_adapter/__init__.py +++ b/benchmarks/harbor/src/harbor_adapter/__init__.py @@ -1,5 +1,6 @@ """Public import for the repository-local nanoPyCodeAgent Harbor adapter.""" from .adapter import NanoPyCodeAgent +from .verifier import RetryingVerifier -__all__ = ["NanoPyCodeAgent"] +__all__ = ["NanoPyCodeAgent", "RetryingVerifier"] diff --git a/benchmarks/harbor/src/harbor_adapter/__main__.py b/benchmarks/harbor/src/harbor_adapter/__main__.py new file mode 100644 index 0000000..2b07ce6 --- /dev/null +++ b/benchmarks/harbor/src/harbor_adapter/__main__.py @@ -0,0 +1,182 @@ +"""Repository benchmark entry point with recorded source and bootstrap policy.""" + +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +from zipfile import ZipFile + +from harbor.models.job.config import JobConfig + +from .bootstrap import AptCache, default_cache_dir + + +def standard_config(raw: dict) -> JobConfig: + """Validate policy before building or creating a benchmark environment.""" + raw = json.loads(json.dumps(raw)) + raw.setdefault("retry", {"max_retries": 0}) + raw.setdefault("n_concurrent_trials", 1) + raw.setdefault("verifier", { + "import_path": "harbor_adapter:RetryingVerifier", + "kwargs": {"max_attempts": 3}, + }) + raw.setdefault("verifier_timeout_multiplier", 4) + config = JobConfig.model_validate(raw) + if config.retry.max_retries != 0: + raise ValueError("The standard workflow disables whole-trial retries") + if config.verifier.import_path != "harbor_adapter:RetryingVerifier": + raise ValueError("The standard workflow requires RetryingVerifier") + attempts = config.verifier.kwargs.get("max_attempts", 3) + if type(attempts) is not int or attempts != 3: + raise ValueError("The standard workflow requires max_attempts=3") + if ( + config.verifier_timeout_multiplier != 4 + or config.verifier.override_timeout_sec is not None + or config.verifier.max_timeout_sec is not None + ): + raise ValueError( + "Use native per-attempt timeouts and verifier_timeout_multiplier=4" + ) + if config.verifier.disable or config.source_jobs: + raise ValueError("The standard workflow runs fresh trials with verification enabled") + if not config.tasks and not config.datasets: + raise ValueError("Select at least one task or dataset") + if not config.agents or config.n_attempts < 1: + raise ValueError("Select at least one agent and one trial attempt") + for agent in config.agents: + identifiers = [value for value in (agent.import_path, agent.name) if value] + if not identifiers or any( + value != "harbor_adapter:NanoPyCodeAgent" for value in identifiers + ): + raise ValueError("The standard workflow requires harbor_adapter:NanoPyCodeAgent") + if agent.env or set(agent.kwargs) - { + "max_turns", "max_tokens", "time_budget_seconds", + }: + raise ValueError( + "Pass model credentials through the environment; " + "agent kwargs may only set budgets" + ) + if not agent.model_name: + raise ValueError("Each agent requires an explicit model_name") + return config + + +def summarize(job_dir: Path, *, install_only: bool = False) -> dict: + trials = [] + for path in sorted(job_dir.glob("*/result.json")): + result = json.loads(path.read_text()) + bootstrap_path = path.parent / "agent/bootstrap.json" + bootstrap = json.loads(bootstrap_path.read_text()) if bootstrap_path.exists() else {} + exception = result.get("exception_info") + verifier = result.get("verifier_result") or {} + retry_path = path.parent / "verifier/retry-summary.json" + if exception: + status = "setup_failed" if bootstrap.get("status") == "failed" else "execution_error" + elif install_only and bootstrap.get("status") == "ready": + status = "setup_passed" + else: + status = "scored" if verifier.get("rewards") is not None else "unscored" + trials.append({ + "task": result.get("task_name"), "trial": path.parent.name, + "status": status, + "exception_type": exception.get("exception_type") if exception else None, + "rewards": None if exception else verifier.get("rewards"), + "bootstrap": bootstrap, + "verifier_attempts": json.loads(retry_path.read_text()) if retry_path.exists() else None, + }) + return {"trials": trials} + + +def run(args) -> int: + root = Path.cwd() + if not (root / "src/nanopycodeagent").is_dir(): + raise ValueError("Run from the nanoPyCodeAgent repository root") + config = standard_config(json.loads(args.config.read_text())) + if args.job_name: + config.job_name = args.job_name + if Path(config.job_name).name != config.job_name or config.job_name in {".", ".."}: + raise ValueError("job_name must be a single directory name") + config.install_only = args.install_only or config.install_only + config.jobs_dir = config.jobs_dir.resolve() + job_dir = config.jobs_dir / config.job_name + record = config.jobs_dir / (config.job_name + "-input") + if job_dir.exists() or record.exists(): + raise ValueError("Use a new job_name; existing jobs and inputs are never overwritten") + record.mkdir(parents=True) + record.chmod(0o700) + source = record / "source" + shutil.copytree(root / "benchmarks/harbor/src/harbor_adapter", source / "harbor_adapter", ignore=shutil.ignore_patterns("__pycache__")) + shutil.copytree(root / "src/nanopycodeagent", source / "nanopycodeagent", ignore=shutil.ignore_patterns("__pycache__")) + subprocess.run(["uv", "build", "--wheel", "--out-dir", str(record / "dist")], check=True) + wheel, = (record / "dist").glob("*.whl") + with ZipFile(wheel) as archive: + for path in (source / "nanopycodeagent").rglob("*.py"): + if archive.read(path.relative_to(source).as_posix()) != path.read_bytes(): + raise ValueError("Source changed during preparation; prepare a new job") + cache = args.cache_dir.expanduser().resolve() + for agent in config.agents: + agent.kwargs.update(wheel_path=str(wheel), bootstrap_cache=str(cache)) + config_path = record / "config.json" + config_path.write_text(config.model_dump_json(indent=2) + "\n") + manifest = { + "created_at": datetime.now(timezone.utc).isoformat(), + "commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + "branch": subprocess.check_output(["git", "branch", "--show-current"], text=True).strip(), + "source_kind": "working tree including uncommitted files", + "source_sha256": {str(p.relative_to(source)): hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(source.rglob("*.py"))}, + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), + "cache_dir": str(cache), + } + (record / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + (record / "working-tree.patch").write_bytes(subprocess.check_output(["git", "diff", "HEAD"])) + print(f"Prepared {record}", flush=True) + if args.prepare_only: + return 0 + env = dict(os.environ) + env["PYTHONPATH"] = str(source) + env["PATH"] = str(Path(sys.executable).parent) + os.pathsep + env["PATH"] + code = None + try: + with (record / "harbor.log").open("w") as log: + code = subprocess.run( + ["harbor", "run", "--config", str(config_path), "--yes"], + env=env, stdout=log, stderr=subprocess.STDOUT, + ).returncode + finally: + summary = summarize(job_dir, install_only=config.install_only) + summary["harbor_exit_code"] = code + (record / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(f"Results: {record / 'summary.json'}", flush=True) + incomplete = not summary["trials"] or any( + row["status"] not in {"scored", "setup_passed"} for row in summary["trials"] + ) + return code or int(incomplete) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + benchmark = commands.add_parser("run", help="Record the working tree and run the standard Harbor workflow") + benchmark.add_argument("--config", type=Path, required=True) + benchmark.add_argument("--job-name") + benchmark.add_argument("--cache-dir", type=Path, default=default_cache_dir()) + benchmark.add_argument("--prepare-only", action="store_true", help="Build and record inputs without starting containers or calling the model") + benchmark.add_argument("--install-only", action="store_true", help="Run setup and dependency preflight without calling the model or verifier") + cache = commands.add_parser("import-apt", help="Import a manifest and its sibling packages/ directory") + cache.add_argument("manifest", type=Path) + cache.add_argument("--cache-dir", type=Path, default=default_cache_dir()) + args = parser.parse_args() + if args.command == "import-apt": + count = AptCache(args.cache_dir.expanduser().resolve()).import_manifest(args.manifest) + print(f"Imported {count} verified APT packages into {args.cache_dir}") + return 0 + return run(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py b/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py new file mode 100644 index 0000000..da4f84c --- /dev/null +++ b/benchmarks/harbor/src/harbor_adapter/_verifier_runner.py @@ -0,0 +1,143 @@ +"""Run a Linux verifier inside its container with bounded timeout retries. + +This file is sent to the container as Python source and uses only the standard +library. It never starts an agent or changes the task's test script. +""" + +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess +import sys +import time + + +def _stop_process(process): + # GNU timeout and other tools may create additional process groups in the + # test's session. Snapshot them before killing the session leader. + members = [] + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + fields = (entry / "stat").read_text().rsplit(")", 1)[1].split() + if int(fields[3]) == process.pid: + members.append(int(entry.name)) + except (OSError, ValueError, IndexError): + continue + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + for pid in members: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) + + +def _archive(logs, attempt): + target = logs / "attempts" / str(attempt) + target.mkdir(parents=True) + for source in logs.iterdir(): + if source.name in {"attempts", "retry-summary.json"}: + continue + if source.is_dir(): + shutil.copytree(source, target / source.name) + else: + shutil.copy2(source, target / source.name) + + +def run(command, timeout_sec, max_attempts, logs, control): + """Return execution status; only our own elapsed deadline is retryable.""" + logs = Path(logs) + control = Path(control) + cancelled = control.with_suffix(".cancelled") + attempts = [] + summary = {"timeout_sec": timeout_sec, "max_attempts": max_attempts, + "attempts": attempts} + + def interrupted(signum, frame): + raise InterruptedError("Verifier execution cancelled") + + signal.signal(signal.SIGTERM, interrupted) + signal.signal(signal.SIGINT, interrupted) + control.write_text(str(os.getpid())) + try: + for attempt in range(1, max_attempts + 1): + if cancelled.exists(): + raise InterruptedError("Verifier execution cancelled") + # A timed-out test may have written a reward before hanging. It + # must never be mistaken for the next attempt's result. + for previous in logs.iterdir(): + if previous.name in {"attempts", "retry-summary.json"}: + continue + if previous.is_dir(): + shutil.rmtree(previous) + else: + previous.unlink() + started = time.monotonic() + record = {"attempt": attempt, "status": "error"} + attempts.append(record) + process = None + try: + process = subprocess.Popen( + ["bash", "-c", command], stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + start_new_session=True, + ) + try: + record["return_code"] = process.wait(timeout=timeout_sec) + record["status"] = "completed" + except subprocess.TimeoutExpired: + _stop_process(process) + record["status"] = "timeout" + except BaseException: + if process is not None: + _stop_process(process) + record["status"] = "cancelled" + raise + finally: + record["duration_seconds"] = time.monotonic() - started + _archive(logs, attempt) + # Remove even the last timed-out attempt's reward from the + # canonical location; the original stays in its archive. + if record["status"] != "completed": + for name in ("reward.txt", "reward.json"): + (logs / name).unlink(missing_ok=True) + (logs / "retry-summary.json").write_text(json.dumps(summary, indent=2)) + if record["status"] == "completed": + return record + return attempts[-1] + finally: + control.unlink(missing_ok=True) + + +def cancel(control): + """Stop this runner, or prevent it starting after a transport cancellation.""" + control = Path(control) + control.with_suffix(".cancelled").touch() + try: + pid = int(control.read_text()) + except FileNotFoundError: + return + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + return + for _ in range(100): + if not control.exists(): + return + time.sleep(0.05) + raise RuntimeError("Verifier runner did not stop after cancellation") + + +if __name__ == "__main__": + if sys.argv[1] == "cancel": + cancel(sys.argv[2]) + else: + command, timeout, count, logs, control = sys.argv[1:] + print(json.dumps(run(command, float(timeout), int(count), logs, control))) diff --git a/benchmarks/harbor/src/harbor_adapter/adapter.py b/benchmarks/harbor/src/harbor_adapter/adapter.py index 94141ce..8d92253 100644 --- a/benchmarks/harbor/src/harbor_adapter/adapter.py +++ b/benchmarks/harbor/src/harbor_adapter/adapter.py @@ -1,6 +1,9 @@ """Harbor adapter for running nanoPyCodeAgent in benchmark containers.""" import asyncio +import hashlib +import json +from pathlib import Path import re import shlex import uuid @@ -17,11 +20,12 @@ from harbor.models.agent.context import AgentContext from harbor.models.trajectories.trajectory import Trajectory +from .bootstrap import AptCache, default_cache_dir, verifier_profile + _DEFAULT_MAX_TURNS = 50 _SYSTEM_DEPENDENCIES = ("curl", "bash", "git", "python3", "ca_certificates") -# Pinned task images occasionally resolve a package index that still points at -# removed `.deb` versions, producing a transient 404 during `apt-get install`. -# Retry the whole transaction a couple of times before failing setup. +# A 404 can reflect a stale index or an unavailable file. Restore verified +# downloads first, then retry the package transaction at most twice. _DEPENDENCY_RETRY_DELAYS = (2.0, 4.0) _PACKAGE_NAME = "nanoPyCodeAgent" _REPOSITORY_URL = "https://github.com/minixalpha/nanoPyCodeAgent.git" @@ -49,7 +53,10 @@ class NanoPyCodeAgent(BaseInstalledAgent): CliFlag("time_budget_seconds", cli="--time-budget-seconds", type="int"), ] - def __init__(self, *args, git_ref: str | None = None, **kwargs): + def __init__( + self, *args, git_ref: str | None = None, wheel_path: str | None = None, + bootstrap_cache: str | None = None, **kwargs, + ): if git_ref is not None: if not isinstance(git_ref, str): raise ValueError( @@ -59,9 +66,22 @@ def __init__(self, *args, git_ref: str | None = None, **kwargs): git_ref = git_ref.strip() if not git_ref: raise ValueError("git_ref must not be blank") - if git_ref is not None and kwargs.get("version") is not None: - raise ValueError("version and git_ref are mutually exclusive") + pins = (git_ref, wheel_path, kwargs.get("version")) + if sum(pin is not None for pin in pins) > 1: + raise ValueError("version, git_ref, and wheel_path are mutually exclusive") self._git_ref = git_ref + self._wheel_path = ( + Path(wheel_path).expanduser().resolve() if wheel_path else None + ) + if self._wheel_path is not None and ( + not self._wheel_path.is_file() or self._wheel_path.suffix != ".whl" + ): + raise ValueError("wheel_path must point to an existing wheel") + self._bootstrap_cache = AptCache( + Path(bootstrap_cache).expanduser().resolve() + if bootstrap_cache else default_cache_dir() + ) + self._bootstrap_record: dict = {} super().__init__(*args, **kwargs) max_tokens = self._resolved_flags.get("max_tokens") if max_tokens is not None and max_tokens < 1: @@ -85,6 +105,8 @@ def parse_version(self, stdout: str) -> str: return match.group(1) if match else lines[-1] def _install_target(self) -> str: + if self._wheel_path is not None: + return "/tmp/" + self._wheel_path.name if self._git_ref is not None: return f"git+{_REPOSITORY_URL}@{self._git_ref}" if self._version is not None: @@ -93,7 +115,49 @@ def _install_target(self) -> str: @override async def install(self, environment: BaseEnvironment) -> None: - await self._ensure_system_dependencies(environment) + record = self._bootstrap_record + record.update( + status="running", stage="profile", + cache_dir=str(self._bootstrap_cache.root), + ) + try: + profile = verifier_profile( + self.logs_dir, getattr(environment, "environment_name", None), + ) + record["verifier_preflight"] = { + "profile": profile.name if profile else None, + "status": "pending" if profile else "not_configured", + } + record["stage"] = "system_dependencies" + await self._ensure_system_dependencies(environment) + record["stage"] = "agent_install" + await self._install_agent(environment) + if profile: + record["stage"] = "verifier_dependencies" + await self._install_apt(environment, profile.apt_packages) + await self.exec_as_agent(environment, command=profile.command) + record["verifier_preflight"]["status"] = "passed" + record.update(status="ready", stage="complete") + except BaseException as exc: + record.update(status="failed", error_type=type(exc).__name__) + if record.get("verifier_preflight", {}).get("status") == "pending": + record["verifier_preflight"]["status"] = "incomplete" + raise + finally: + record["cache_used"] = any( + a["restored"] for a in record.get("apt", []) + ) + self.logs_dir.mkdir(parents=True, exist_ok=True) + (self.logs_dir / "bootstrap.json").write_text( + json.dumps(record, indent=2) + "\n" + ) + + async def _install_agent(self, environment: BaseEnvironment) -> None: + if self._wheel_path is not None: + await environment.upload_file(self._wheel_path, self._install_target()) + self._bootstrap_record["wheel_sha256"] = hashlib.sha256( + self._wheel_path.read_bytes() + ).hexdigest() install_target = shlex.quote(self._install_target()) await self.exec_as_agent( environment, @@ -116,12 +180,29 @@ async def _ensure_system_dependencies( The retry only covers the system-package step, so it cannot replay any model work; a non-404 failure or the final attempt propagates. """ + if await self._get_system_package_manager(environment) == "apt-get": + packages = tuple( + package for dependency in _SYSTEM_DEPENDENCIES + for package in self.SYSTEM_PACKAGES[dependency].packages["apt-get"] + ) + await self._install_apt(environment, packages) + return + await self._retry_dependency_install( + lambda: self.ensure_system_dependencies(environment, _SYSTEM_DEPENDENCIES) + ) + + async def _install_apt(self, environment, packages) -> None: + await self._retry_dependency_install( + lambda: self._bootstrap_cache.install( + self, environment, packages, self._bootstrap_record, + ) + ) + + async def _retry_dependency_install(self, install) -> None: total_attempts = len(_DEPENDENCY_RETRY_DELAYS) + 1 for attempt, delay in enumerate((*_DEPENDENCY_RETRY_DELAYS, None)): try: - await self.ensure_system_dependencies( - environment, _SYSTEM_DEPENDENCIES - ) + await install() return except NonZeroAgentExitCodeError as error: if delay is None or "404" not in str(error): @@ -185,6 +266,9 @@ async def run( @override def populate_context_post_run(self, context: AgentContext) -> None: + if self._bootstrap_record: + context.metadata = context.metadata or {} + context.metadata["bootstrap"] = self._bootstrap_record trajectory_path = self.logs_dir / "trajectory.json" try: trajectory = Trajectory.model_validate_json( @@ -223,6 +307,8 @@ def populate_context_post_run(self, context: AgentContext) -> None: usage_complete=extra.get("usage_complete"), cost_is_partial=extra.get("cost_is_partial"), known_cost_usd=extra.get("known_cost_usd"), + cost_is_estimated=extra.get("cost_is_estimated"), + estimated_cost_usd=extra.get("estimated_cost_usd"), missing_generation_ids=extra.get("missing_generation_ids"), ) if metrics is None: diff --git a/benchmarks/harbor/src/harbor_adapter/bootstrap.py b/benchmarks/harbor/src/harbor_adapter/bootstrap.py new file mode 100644 index 0000000..845f6ce --- /dev/null +++ b/benchmarks/harbor/src/harbor_adapter/bootstrap.py @@ -0,0 +1,179 @@ +"""Verified, reusable APT downloads and versioned verifier dependency profiles.""" + +from dataclasses import asdict, dataclass +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import shutil +import tempfile + + +def default_cache_dir() -> Path: + return Path( + os.environ.get("NANOPY_HARBOR_CACHE", ".cache/nanopy-harbor") + ).expanduser().resolve() + + +@dataclass(frozen=True) +class Package: + url: str + filename: str + size: int + sha256: str + + def __post_init__(self): + if not re.fullmatch(r"[A-Za-z0-9.+%~:_-]+\.deb", self.filename): + raise ValueError(f"Unsafe package filename: {self.filename!r}") + if type(self.size) is not int or self.size <= 0: + raise ValueError("Package size must be a positive integer") + if not re.fullmatch(r"[a-f0-9]{64}", self.sha256): + raise ValueError("Package requires a SHA256 digest") + + def verify(self, path: Path) -> None: + with path.open("rb") as source: + digest = hashlib.file_digest(source, "sha256").hexdigest() + if path.stat().st_size != self.size or digest != self.sha256: + raise ValueError(f"Package checksum/size mismatch: {self.filename}") + + +def parse_apt_plan(stdout: str) -> list[Package]: + packages = [] + for line in stdout.splitlines(): + if not line.startswith("'"): + continue + url, filename, size, digest = shlex.split(line) + if not digest.startswith("SHA256:"): + raise ValueError("APT download plan did not provide SHA256 hashes") + packages.append( + Package(url, filename, int(size), digest.removeprefix("SHA256:")) + ) + return packages + + +class AptCache: + """Key downloads by their APT-index hash, never by a mutable package name.""" + + def __init__(self, root: Path): + self.root = root + + def path(self, package: Package) -> Path: + return self.root / "apt" / package.sha256 / package.filename + + def store(self, package: Package, source: Path) -> None: + package.verify(source) + target = self.path(package) + target.parent.mkdir(parents=True, exist_ok=True) + # Atomic publication also supports concurrent trials sharing this cache. + with tempfile.TemporaryDirectory(dir=target.parent) as temporary: + staged = Path(temporary) / package.filename + shutil.copyfile(source, staged) + package.verify(staged) + staged.replace(target) + + def import_manifest(self, manifest: Path) -> int: + packages = [Package(**row) for row in json.loads(manifest.read_text())] + for package in packages: + package.verify(manifest.parent / "packages" / package.filename) + for package in packages: + self.store(package, manifest.parent / "packages" / package.filename) + return len(packages) + + async def install( + self, agent, environment, packages: tuple[str, ...], record: dict, + ) -> None: + """Restore only files selected by the current index; let APT install them. + + Download before installation so Docker's APT cleanup hooks cannot remove + successful downloads before they enter the shared cache. + """ + args = shlex.join(packages) + env = {"DEBIAN_FRONTEND": "noninteractive"} + await agent.exec_as_root(environment, command="apt-get update", env=env) + result = await agent.exec_as_root( + environment, + command=( + "apt-get -o Acquire::ForceHash=sha256 --print-uris " + f"-y --download-only install {args}" + ), + env=env, + ) + plan = parse_apt_plan(result.stdout) + attempt = { + "requested": list(packages), "packages": [asdict(p) for p in plan], + "restored": [], "saved": [], + } + record.setdefault("apt", []).append(attempt) + for package in plan: + cached = self.path(package) + if cached.exists(): + package.verify(cached) + await environment.upload_file( + cached, f"/var/cache/apt/archives/{package.filename}", + ) + attempt["restored"].append(package.filename) + await agent.exec_as_root( + environment, command=f"apt-get -y --download-only install {args}", env=env, + ) + with tempfile.TemporaryDirectory(prefix="nanopy-apt-") as temporary: + for package in plan: + if package.filename in attempt["restored"]: + continue + downloaded = Path(temporary) / package.filename + await environment.download_file( + source_path=f"/var/cache/apt/archives/{package.filename}", + target_path=downloaded, + ) + self.store(package, downloaded) + attempt["saved"].append(package.filename) + await agent.exec_as_root( + environment, command=f"apt-get install -y {args}", env=env, + ) + + +# Profiles describe dependencies only. Neither task solutions nor assertions are +# loaded during setup. A task revision change requires reviewing its profile. +QEMU_TASK = "terminal-bench/qemu-alpine-ssh" +QEMU_REF = "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28" +QEMU_PREFLIGHT = ( + "set -euo pipefail; " + "curl -LsSf https://astral.sh/uv/0.9.5/install.sh | sh; " + 'export PATH="$HOME/.local/bin:$PATH"; ' + "uv --version | grep -Eq '^uv 0\\.9\\.5( |$)'; " + "command -v sshpass; " + "uvx -p 3.13 -w pytest==8.4.1 -w pytest-json-ctrf==0.3.5 pytest --version" +) + + +@dataclass(frozen=True) +class VerifierProfile: + name: str + apt_packages: tuple[str, ...] + command: str + + +VERIFIER_PROFILES = { + (QEMU_TASK, QEMU_REF): VerifierProfile( + "qemu-alpine-ssh-v1", ("curl", "sshpass"), QEMU_PREFLIGHT, + ), +} + + +def verifier_profile( + logs_dir: Path, environment_name: str | None, +) -> VerifierProfile | None: + known_tasks = {name for name, _ in VERIFIER_PROFILES} + config_path = logs_dir.parent / "config.json" + if not config_path.exists(): + if environment_name in {name.split("/")[-1] for name in known_tasks}: + raise ValueError("Verifier preflight requires Harbor's pinned trial config") + return None + task = json.loads(config_path.read_text())["task"] + profile = VERIFIER_PROFILES.get((task.get("name"), task.get("ref"))) + if profile is None and task.get("name") in known_tasks: + raise ValueError( + f"Unknown {task['name']} revision; review its verifier dependency profile" + ) + return profile diff --git a/benchmarks/harbor/src/harbor_adapter/verifier.py b/benchmarks/harbor/src/harbor_adapter/verifier.py new file mode 100644 index 0000000..930310b --- /dev/null +++ b/benchmarks/harbor/src/harbor_adapter/verifier.py @@ -0,0 +1,139 @@ +"""Harbor's script verifier with up to three attempts after a timeout.""" + +import asyncio +import json +import math +from pathlib import Path +import shlex +import uuid + +from harbor.models.task.config import TaskOS +from harbor.models.trial.config import TrialConfig +from harbor.models.trial.paths import EnvironmentPaths +from harbor.utils.scripts import build_execution_command +from harbor.verifier.verifier import Verifier + +_RUNNER = Path(__file__).with_name("_verifier_runner.py").read_text() + + +class _RetryEnvironment: + """Delegate Harbor operations, wrapping only the test-script execution.""" + + def __init__(self, environment, command, timeout_sec, max_attempts, logger): + self._environment = environment + self._command = command + self._timeout_sec = timeout_sec + self._max_attempts = max_attempts + self._logger = logger + + def __getattr__(self, name): + return getattr(self._environment, name) + + async def exec(self, *, command, **kwargs): + if command != self._command: + return await self._environment.exec(command=command, **kwargs) + control = f"/tmp/nanopy-verifier-{uuid.uuid4().hex}.pid" + invocation = shlex.join([ + "python3", "-c", _RUNNER, command, str(self._timeout_sec), + str(self._max_attempts), "/logs/verifier", control, + ]) + try: + result = await self._environment.exec(command=invocation, **kwargs) + except BaseException: + # Cancelling a Docker exec client alone can leave the command + # alive inside the container. Stop the runner and its test session. + try: + cleanup = await asyncio.wait_for(self._environment.exec( + command=shlex.join(["python3", "-c", _RUNNER, "cancel", control]), + user="root", + ), timeout=10) + if cleanup.return_code != 0: + raise RuntimeError(cleanup.stderr) + except Exception: + self._logger.exception("Failed to stop the verifier retry runner") + raise + if result.return_code != 0: + raise RuntimeError(f"Verifier retry runner failed: {result.stderr}") + record = json.loads(result.stdout) + if record["status"] == "timeout": + raise TimeoutError( + f"Verifier timed out after {self._max_attempts} attempts " + f"of {self._timeout_sec:g} seconds" + ) + return result + + +class RetryingVerifier(Verifier): + """Retry native Linux test-script timeouts without rerunning the agent. + + Select with ``--verifier harbor_adapter:RetryingVerifier`` and allow the + overall verification phase enough time via ``--verifier-timeout-multiplier + 4``. Each attempt retains the task's native verifier timeout. + """ + + def __init__(self, *, max_attempts=3, **kwargs): + if type(max_attempts) is not int or not 1 <= max_attempts <= 3: + raise ValueError("max_attempts must be an integer between 1 and 3") + super().__init__(**kwargs) + if self.environment.os != TaskOS.LINUX or self.step_name is not None: + raise ValueError("RetryingVerifier supports single-step Linux tasks") + self._max_attempts = max_attempts + self._attempt_timeout = self.task.config.verifier.timeout_sec + if not math.isfinite(self._attempt_timeout) or self._attempt_timeout <= 0: + raise ValueError("The task's verifier timeout must be finite and positive") + + async def verify(self): + # Harbor's outer watchdog covers the entire custom verifier. Reject an + # insufficient budget instead of silently losing the promised retries. + config = TrialConfig.model_validate_json(self.trial_paths.config_path.read_text()) + multiplier = config.verifier_timeout_multiplier + if multiplier is None: + multiplier = config.timeout_multiplier + base_timeout = ( + config.verifier.override_timeout_sec or self._attempt_timeout + ) + outer_timeout = min( + base_timeout, config.verifier.max_timeout_sec or float("inf"), + ) * multiplier + if outer_timeout <= self._attempt_timeout * self._max_attempts: + raise ValueError( + "Allow time for every verifier attempt and log collection: use " + "--verifier-timeout-multiplier 4 and remove conflicting timeout caps" + ) + _, tests_source_dir, host_test_path = self._resolve_tests() + paths = EnvironmentPaths.for_os(self.environment.os) + command = build_execution_command( + str(paths.tests_dir / host_test_path.relative_to(tests_source_dir).as_posix()), + stdout_path=str(paths.verifier_dir / "test-stdout.txt"), + task_os=self.environment.os, + ) + environment = self.environment + self.environment = _RetryEnvironment( + environment, command, self._attempt_timeout, self._max_attempts, self.logger, + ) + try: + return await super().verify() + finally: + self.environment = environment + # The standard verifier downloads logs only after successful exec. + # Preserve failed attempts too, even on non-mounted environments. + if not environment.capabilities.mounted: + try: + if self.include_logs or self.exclude_logs: + download = environment.download_dir_filtered( + source_dir=str(paths.verifier_dir), + target_dir=self.trial_paths.verifier_dir, + include=self.include_logs or None, + exclude=self.exclude_logs or None, + protect=["reward.txt", "reward.json", "attempts/**", "retry-summary.json"], + ) + else: + download = environment.download_dir( + source_dir=str(paths.verifier_dir), + target_dir=self.trial_paths.verifier_dir, + ) + await asyncio.wait_for(download, timeout=30) + except Exception: + # Collection failure must not hide the original timeout, + # cancellation, or reward parsing error. + self.logger.exception("Failed to collect verifier attempt logs") diff --git a/benchmarks/harbor/tests/test_atif_compatibility.py b/benchmarks/harbor/tests/test_atif_compatibility.py index 4456078..d4ca37b 100644 --- a/benchmarks/harbor/tests/test_atif_compatibility.py +++ b/benchmarks/harbor/tests/test_atif_compatibility.py @@ -1,11 +1,15 @@ """Compatibility checks against Harbor's pinned ATIF-v1.7 validator.""" +import json from pathlib import Path import pytest +from harbor.models.agent.context import AgentContext from harbor.utils.trajectory_validator import TrajectoryValidator from nanopycodeagent.atif import project_atif +from nanopycodeagent.cost import estimated_cost, usage_cost from nanopycodeagent.event_journal import EventJournal, NativeEvent +from harbor_adapter import NanoPyCodeAgent def test_projector_output_passes_harbor_atif_validator(): @@ -16,6 +20,54 @@ def test_projector_output_passes_harbor_atif_validator(): assert validator.validate(trajectory), validator.get_errors() +@pytest.mark.parametrize("other_kind", [None, "estimated", "provider_reported", "pending"]) +def test_estimated_costs_survive_projection_validation_and_harbor_context(tmp_path, other_kind): + fixture = Path(__file__).parent / "fixtures" / "atif-journal-v1.jsonl" + usage = {"input_tokens": 1000, "output_tokens": 1000} + with EventJournal.create("run-estimated", directory=tmp_path) as journal: + for entry in EventJournal.replay(fixture): + payload = entry.payload + if entry.type == "model.completed": + payload = payload | { + "generation_id": None, + "cost": estimated_cost("deepseek-flash", usage), + } + journal.append(NativeEvent(entry.type, payload)) + if entry.type == "model.completed" and other_kind is not None: + other_cost = { + "estimated": estimated_cost("deepseek-flash", usage), + "provider_reported": usage_cost({"cost": "0.002"}), + "pending": {"status": "pending", "source": "provider_generation"}, + }[other_kind] + journal.append(NativeEvent("model.completed", payload | { + "model_call_id": "model-2", "generation_id": "generation-2", + "message_id": "msg-2", "stop_reason": "end_turn", + "content": [{"type": "text", "text": "done"}], + "tool_calls": [], "cost": other_cost, + })) + + trajectory = project_atif(EventJournal.replay(journal.path)) + validator = TrajectoryValidator() + assert validator.validate(trajectory), validator.get_errors() + (tmp_path / "trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8") + context = AgentContext() + NanoPyCodeAgent(logs_dir=tmp_path).populate_context_post_run(context) + + diagnostic = context.metadata["trajectory"] + assert diagnostic["cost_is_estimated"] is True + assert diagnostic["estimated_cost_usd"] == (0.003 if other_kind == "estimated" else 0.0015) + if other_kind == "pending": + assert context.cost_usd is None + assert diagnostic["status"] == "partial" + assert diagnostic["known_cost_usd"] == 0.0015 + assert diagnostic["missing_generation_ids"] == ["generation-2"] + else: + assert context.cost_usd == { + None: 0.0015, "estimated": 0.003, "provider_reported": 0.0035, + }[other_kind] + assert diagnostic["status"] == "complete" + + def test_time_budget_v4_passes_harbor_atif_validator(tmp_path): fixture = Path(__file__).parent / "fixtures" / "atif-journal-v1.jsonl" note = "[time budget] Stop investigating now. Write your best answer to the output file." diff --git a/benchmarks/harbor/tests/test_bootstrap.py b/benchmarks/harbor/tests/test_bootstrap.py new file mode 100644 index 0000000..b11bec1 --- /dev/null +++ b/benchmarks/harbor/tests/test_bootstrap.py @@ -0,0 +1,186 @@ +"""Cache integrity and setup-before-model contracts for benchmark bootstrap.""" + +import asyncio +from dataclasses import asdict +import hashlib +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from harbor_adapter import NanoPyCodeAgent +from harbor_adapter.bootstrap import AptCache, Package, QEMU_REF, QEMU_TASK, parse_apt_plan, verifier_profile + + +def package(data=b"verified deb", filename="curl_1_amd64.deb"): + return Package("https://deb.example/" + filename, filename, len(data), hashlib.sha256(data).hexdigest()) + + +def apt_line(item): + return f"'{item.url}' {item.filename} {item.size} SHA256:{item.sha256}" + + +class AptEnvironment: + def __init__(self, item, data): + self.item = item + self.data = data + self.uploaded = [] + self.downloaded = [] + self.commands = [] + + async def upload_file(self, source, target): + self.uploaded.append((Path(source).read_bytes(), target)) + + async def download_file(self, source_path, target_path): + self.downloaded.append(source_path) + Path(target_path).write_bytes(self.data) + + async def exec_as_root(self, environment, command, **kwargs): + self.commands.append(command) + return SimpleNamespace(stdout=apt_line(self.item) if "--print-uris" in command else "") + + +def test_restores_only_the_current_index_hash_and_saves_downloads(tmp_path): + cache = AptCache(tmp_path) + data = b"current package" + item = package(data) + stale = package(b"old package") + source = tmp_path / "download.deb" + source.write_bytes(b"old package") + cache.store(stale, source) + first = AptEnvironment(item, data) + record = {} + asyncio.run(cache.install(first, first, ("curl",), record)) + assert first.uploaded == [] + assert len(first.downloaded) == 1 + assert cache.path(item).read_bytes() == data + assert record["apt"][0]["saved"] == [item.filename] + assert first.commands[-1] == "apt-get install -y curl" + + second = AptEnvironment(item, data) + record = {} + asyncio.run(cache.install(second, second, ("curl",), record)) + assert second.uploaded == [(data, "/var/cache/apt/archives/" + item.filename)] + assert second.downloaded == [] + assert record["apt"][0]["restored"] == [item.filename] + + +def test_corrupt_cache_fails_before_upload_or_install(tmp_path): + item = package() + cache = AptCache(tmp_path) + cache.path(item).parent.mkdir(parents=True) + cache.path(item).write_bytes(b"tampered") + environment = AptEnvironment(item, b"verified deb") + with pytest.raises(ValueError, match="checksum/size"): + asyncio.run(cache.install(environment, environment, ("curl",), {})) + assert not environment.uploaded + assert not any(command.startswith("apt-get install") for command in environment.commands) + + +def test_invalid_download_is_never_published_or_installed(tmp_path): + item = package() + cache = AptCache(tmp_path) + environment = AptEnvironment(item, b"corrupt response") + with pytest.raises(ValueError, match="checksum/size"): + asyncio.run(cache.install(environment, environment, ("curl",), {})) + assert not cache.path(item).exists() + assert not any(command.startswith("apt-get install") for command in environment.commands) + + +def test_import_validates_all_packages_before_publishing(tmp_path): + source = tmp_path / "old" + (source / "packages").mkdir(parents=True) + first = package() + second = package(b"another", "git_1_amd64.deb") + (source / "packages" / first.filename).write_bytes(b"verified deb") + (source / "packages" / second.filename).write_bytes(b"bad") + manifest = source / "manifest.json" + manifest.write_text(json.dumps([asdict(first), asdict(second)])) + cache = AptCache(tmp_path / "cache") + with pytest.raises(ValueError, match="checksum/size"): + cache.import_manifest(manifest) + assert not cache.root.exists() + (source / "packages" / second.filename).write_bytes(b"another") + assert cache.import_manifest(manifest) == 2 + assert cache.path(second).read_bytes() == b"another" + + +@pytest.mark.parametrize("filename", ["../escape.deb", "/tmp/escape.deb", "$(id).deb"]) +def test_manifest_rejects_unsafe_filenames(filename): + with pytest.raises(ValueError, match="Unsafe"): + package(filename=filename) + + +def test_apt_parser_requires_strong_hashes_and_handles_epoch_filenames(): + item = package(filename="git_1%3a2.30.2-1+deb11u5_amd64.deb") + assert parse_apt_plan("Reading package lists...\n" + apt_line(item)) == [item] + with pytest.raises(ValueError, match="SHA256"): + parse_apt_plan(apt_line(item).replace("SHA256:", "MD5Sum:")) + + +def write_task(logs_dir, ref=QEMU_REF): + logs_dir.mkdir(parents=True) + (logs_dir.parent / "config.json").write_text(json.dumps({"task": {"name": QEMU_TASK, "ref": ref}})) + + +def test_profile_is_scoped_to_reviewed_task_revision(tmp_path): + logs_dir = tmp_path / "agent" + write_task(logs_dir) + assert verifier_profile(logs_dir, "qemu-alpine-ssh").name == "qemu-alpine-ssh-v1" + (logs_dir.parent / "config.json").write_text(json.dumps({"task": {"name": QEMU_TASK, "ref": "latest"}})) + with pytest.raises(ValueError, match="Unknown"): + verifier_profile(logs_dir, "qemu-alpine-ssh") + + +def test_preflight_failure_stops_setup_and_records_the_failing_stage(tmp_path, monkeypatch): + logs_dir = tmp_path / "agent" + write_task(logs_dir) + adapter = NanoPyCodeAgent(logs_dir=logs_dir) + calls = [] + + async def dependencies(*args): + calls.append("system") + + async def install(*args): + calls.append("agent_install") + + async def apt(*args): + calls.append("verifier_packages") + + async def fail(*args, command): + calls.append("verifier_preflight") + assert "pytest --version" in command + assert "/tests" not in command + raise RuntimeError("dependency download failed") + + monkeypatch.setattr(adapter, "_ensure_system_dependencies", dependencies) + monkeypatch.setattr(adapter, "_install_agent", install) + monkeypatch.setattr(adapter, "_install_apt", apt) + monkeypatch.setattr(adapter, "exec_as_agent", fail) + with pytest.raises(RuntimeError, match="dependency download"): + asyncio.run(adapter.install(SimpleNamespace(environment_name="qemu-alpine-ssh"))) + assert calls == ["system", "agent_install", "verifier_packages", "verifier_preflight"] + report = json.loads((logs_dir / "bootstrap.json").read_text()) + assert report["status"] == "failed" + assert report["stage"] == "verifier_dependencies" + assert report["verifier_preflight"]["status"] == "incomplete" + + +def test_local_wheel_is_uploaded_and_logged(tmp_path, monkeypatch): + wheel = tmp_path / "nanopycodeagent-0.1-py3-none-any.whl" + wheel.write_bytes(b"wheel contents") + adapter = NanoPyCodeAgent(logs_dir=tmp_path / "logs", wheel_path=str(wheel)) + environment = AptEnvironment(package(), b"") + calls = [] + + async def execute(*args, **kwargs): + calls.append(kwargs["command"]) + + monkeypatch.setattr(adapter, "exec_as_agent", execute) + asyncio.run(adapter._install_agent(environment)) + assert environment.uploaded == [(b"wheel contents", "/tmp/" + wheel.name)] + assert "uv tool install --force /tmp/" + wheel.name in calls[0] + assert adapter._bootstrap_record["wheel_sha256"] == hashlib.sha256(wheel.read_bytes()).hexdigest() + with pytest.raises(ValueError, match="mutually exclusive"): + NanoPyCodeAgent(logs_dir=tmp_path, wheel_path=str(wheel), git_ref="main") diff --git a/benchmarks/harbor/tests/test_verifier.py b/benchmarks/harbor/tests/test_verifier.py new file mode 100644 index 0000000..5ff0372 --- /dev/null +++ b/benchmarks/harbor/tests/test_verifier.py @@ -0,0 +1,259 @@ +"""Execute the retry runner with real processes and Harbor's reward parser.""" + +import asyncio +import json +import os +from pathlib import Path +import shlex +import shutil +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from harbor.environments.base import ExecResult +from harbor.models.task.config import TaskOS +from harbor.models.task.task import Task +from harbor.models.trial.config import TrialConfig, VerifierConfig +from harbor.models.trial.paths import TrialPaths +from harbor.trial.single_step import SingleStepTrial +from harbor.trial.trial import Trial +from harbor.trial.errors import VerifierTimeoutError +from harbor.verifier.factory import VerifierFactory +from harbor.verifier.verifier import RewardFileNotFoundError, VerifierOutputParseError + +from harbor_adapter import RetryingVerifier + + +class LocalEnvironment: + """Map container paths locally while executing the real runner and scripts.""" + + os = TaskOS.LINUX + + def __init__(self, task, paths, mounted): + self.task = task + self.paths = paths + self.capabilities = SimpleNamespace(mounted=mounted) + self.logs = paths.verifier_dir if mounted else paths.trial_dir / "remote-logs" + self.logs.mkdir(exist_ok=True) + self.uploads = 0 + self.downloads = 0 + self.processes = [] + + async def upload_dir(self, **kwargs): + self.uploads += 1 + + async def download_dir(self, *, source_dir, target_dir): + self.downloads += 1 + shutil.copytree(self.logs, target_dir, dirs_exist_ok=True) + + async def exec(self, *, command, env=None, **kwargs): + if command.startswith("chmod +x"): + self.task.paths.test_path.chmod(0o755) + return ExecResult(return_code=0) + args = shlex.split(command) + assert args[:2] == ["python3", "-c"] + if args[3] != "cancel": + args[3] = args[3].replace("/tests/", str(self.task.paths.tests_dir) + "/") + args[3] = args[3].replace("/logs/verifier", str(self.logs)) + args[-2] = str(self.logs) + args[-1] = str(self.paths.trial_dir / Path(args[-1]).name) + process = await asyncio.create_subprocess_exec( + *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + env={**os.environ, "LOGS": str(self.logs), + "STATE": str(self.paths.trial_dir / "count"), + "CHILD_PID": str(self.paths.trial_dir / "child.pid"), **(env or {})}, + ) + self.processes.append(process) + stdout, stderr = await process.communicate() + return ExecResult(return_code=process.returncode, stdout=stdout.decode(), stderr=stderr.decode()) + + +def make_verifier(tmp_path, script, *, mounted=True, max_attempts=3): + task_dir = tmp_path / "task" + (task_dir / "tests").mkdir(parents=True) + (task_dir / "instruction.md").write_text("Write the solution.") + (task_dir / "task.toml").write_text("[verifier]\ntimeout_sec = 0.3\n") + (task_dir / "tests" / "test.sh").write_text("#!/bin/bash\nset -eu\n" + script) + task = Task(task_dir) + paths = TrialPaths(trial_dir=tmp_path / "trial") + paths.mkdir() + config = TrialConfig( + task={"path": task_dir}, verifier_timeout_multiplier=4, + verifier=VerifierConfig( + import_path="harbor_adapter:RetryingVerifier", + kwargs={"max_attempts": max_attempts}, + ), + ) + paths.config_path.write_text(config.model_dump_json()) + environment = LocalEnvironment(task, paths, mounted) + verifier = VerifierFactory.create_verifier_from_config( + config.verifier, task=task, trial_paths=paths, environment=environment, + ) + return verifier, environment, config + + +def summary(verifier): + return json.loads((verifier.trial_paths.verifier_dir / "retry-summary.json").read_text()) + + +@pytest.mark.parametrize("mounted", [True, False]) +def test_timeout_retries_preserve_logs_clear_rewards_and_run_agent_once(tmp_path, mounted): + verifier, environment, config = make_verifier(tmp_path, ''' +count=$(cat "$STATE" 2>/dev/null || echo 0) +count=$((count + 1)) +echo "$count" > "$STATE" +echo "attempt $count" +if [ "$count" -lt 3 ]; then + echo '{"reward": 1}' > "$LOGS/reward.json" + touch "$LOGS/stale.log" + sleep 10 +fi +test ! -e "$LOGS/stale.log" +test ! -e "$LOGS/reward.json" +echo 0 > "$LOGS/reward.txt" +''', mounted=mounted) + # Exercise Harbor's single-step lifecycle with a local verifier and its + # real outer watchdog. No installation, Docker, or model call is needed. + trial = object.__new__(SingleStepTrial) + trial.task = verifier.task + trial.config = config + for name in ("_run_agent", "_upload_agent_logs", "_collect_artifacts", "_stop_agent_environment"): + setattr(trial, name, AsyncMock()) + results = [] + + async def verify(): + results.append(await asyncio.wait_for( + verifier.verify(), timeout=Trial._compute_verifier_timeout_sec(trial), + )) + + trial._run_verifier = verify + asyncio.run(trial._run()) + + trial._run_agent.assert_awaited_once() + assert results[0].rewards == {"reward": 0.0} + assert environment.uploads == 1 + assert [item["status"] for item in summary(verifier)["attempts"]] == [ + "timeout", "timeout", "completed", + ] + logs = verifier.trial_paths.verifier_dir + for attempt in (1, 2, 3): + assert (logs / "attempts" / str(attempt) / "test-stdout.txt").read_text() == f"attempt {attempt}\n" + assert (logs / "attempts/1/reward.json").exists() + assert not (logs / "reward.json").exists() + + +@pytest.mark.parametrize("reward", [0, 1]) +def test_any_valid_score_stops_after_first_attempt(tmp_path, reward): + verifier, _, _ = make_verifier(tmp_path, f'echo {reward} > "$LOGS/reward.txt"\n') + assert asyncio.run(verifier.verify()).rewards == {"reward": reward} + assert len(summary(verifier)["attempts"]) == 1 + + +@pytest.mark.parametrize("mounted", [True, False]) +def test_three_timeouts_leave_no_canonical_reward_and_keep_all_logs(tmp_path, mounted): + verifier, environment, _ = make_verifier(tmp_path, ''' +echo started +echo 1 > "$LOGS/reward.txt" +sleep 10 +''', mounted=mounted) + with pytest.raises(TimeoutError, match="3 attempts"): + asyncio.run(verifier.verify()) + assert len(summary(verifier)["attempts"]) == 3 + assert all(item["status"] == "timeout" for item in summary(verifier)["attempts"]) + assert not verifier.trial_paths.reward_text_path.exists() + assert (verifier.trial_paths.verifier_dir / "attempts/3/test-stdout.txt").read_text() == "started\n" + assert environment.downloads == (0 if mounted else 1) + + +@pytest.mark.parametrize(("script", "error"), [ + ("exit 124\n", RewardFileNotFoundError), + ('echo invalid > "$LOGS/reward.txt"\n', VerifierOutputParseError), +]) +def test_non_timeout_failures_are_not_retried(tmp_path, script, error): + verifier, _, _ = make_verifier(tmp_path, script) + with pytest.raises(error): + asyncio.run(verifier.verify()) + assert len(summary(verifier)["attempts"]) == 1 + assert summary(verifier)["attempts"][0]["status"] == "completed" + + +def process_is_running(pid): + try: + stat = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + except FileNotFoundError: + return False + return stat[0] != "Z" + + +def test_timeout_stops_children_in_additional_process_groups(tmp_path): + verifier, _, _ = make_verifier(tmp_path, ''' +timeout 30 bash -c 'echo $$ > "$CHILD_PID"; sleep 30' & +wait +''', max_attempts=1) + with pytest.raises(TimeoutError, match="1 attempts"): + asyncio.run(verifier.verify()) + pid = int((verifier.trial_paths.trial_dir / "child.pid").read_text()) + assert not process_is_running(pid) + + +def test_cancellation_stops_the_runner_and_never_retries(tmp_path): + verifier, environment, _ = make_verifier(tmp_path, 'echo $$ > "$CHILD_PID"\nsleep 30\n') + + async def cancel(): + verification = asyncio.create_task(verifier.verify()) + pid_file = verifier.trial_paths.trial_dir / "child.pid" + async with asyncio.timeout(5): + while not pid_file.exists(): + await asyncio.sleep(0.01) + verification.cancel() + with pytest.raises(asyncio.CancelledError): + await verification + for process in environment.processes: + await asyncio.wait_for(process.wait(), 2) + + asyncio.run(cancel()) + assert len(summary(verifier)["attempts"]) == 1 + assert summary(verifier)["attempts"][0]["status"] == "cancelled" + pid = int((verifier.trial_paths.trial_dir / "child.pid").read_text()) + assert not process_is_running(pid) + + +@pytest.mark.parametrize("value", [0, 4, -1, True, "3", 1.5]) +def test_invalid_attempt_limits_are_rejected(value): + with pytest.raises(ValueError, match="max_attempts"): + RetryingVerifier(max_attempts=value) + + +def test_outer_watchdog_must_allow_all_attempts(tmp_path): + verifier, environment, config = make_verifier(tmp_path, "exit 0\n") + config.verifier_timeout_multiplier = 1 + verifier.trial_paths.config_path.write_text(config.model_dump_json()) + with pytest.raises(ValueError, match="verifier-timeout-multiplier 4"): + asyncio.run(verifier.verify()) + assert environment.uploads == 0 + + +def test_log_download_failure_preserves_harbor_timeout_error(tmp_path, caplog): + verifier, environment, config = make_verifier( + tmp_path, "sleep 10\n", mounted=False, max_attempts=1, + ) + environment.download_dir = AsyncMock(side_effect=OSError("download failed")) + trial = object.__new__(SingleStepTrial) + trial.task = verifier.task + trial.config = config + trial._verifier_timeout_sec = Trial._compute_verifier_timeout_sec(trial) + trial._result = SimpleNamespace(verifier=None) + trial._emit = AsyncMock() + trial._run_shared_verifier = lambda **kwargs: verifier.verify() + + with pytest.raises(VerifierTimeoutError): + asyncio.run(trial._run_verifier()) + assert "Failed to collect verifier attempt logs" in caplog.text + environment.download_dir.assert_awaited_once() + + +def test_outer_timeout_cap_is_applied_before_multiplier(tmp_path): + verifier, _, config = make_verifier(tmp_path, 'echo 1 > "$LOGS/reward.txt"\n') + config.verifier.max_timeout_sec = 0.3 + verifier.trial_paths.config_path.write_text(config.model_dump_json()) + assert asyncio.run(verifier.verify()).rewards == {"reward": 1} diff --git a/benchmarks/harbor/tests/test_workflow.py b/benchmarks/harbor/tests/test_workflow.py new file mode 100644 index 0000000..94e361f --- /dev/null +++ b/benchmarks/harbor/tests/test_workflow.py @@ -0,0 +1,75 @@ +"""Standard workflow policy and score/infra separation.""" + +import json + +import pytest + +from harbor_adapter.__main__ import standard_config, summarize + + +def config(): + return { + "tasks": [{"name": "terminal-bench/test", "ref": "sha256:abc"}], + "agents": [{"name": "harbor_adapter:NanoPyCodeAgent", "model_name": "test/model"}], + } + + +def test_standard_policy_enables_native_verifier_retries_without_agent_retries(): + result = standard_config(config()) + assert result.n_attempts == 1 + assert result.retry.max_retries == 0 + assert result.verifier.import_path == "harbor_adapter:RetryingVerifier" + assert result.verifier.kwargs == {"max_attempts": 3} + assert result.verifier_timeout_multiplier == 4 + + +@pytest.mark.parametrize("change", [ + {"retry": {"max_retries": 1}}, + {"verifier_timeout_multiplier": 1}, + {"verifier": {"disable": True}}, + {"verifier": {"import_path": "custom:Verifier"}}, + {"agents": [{"name": "custom:Agent", "model_name": "test/model"}]}, + {"agents": [{"name": "custom:Agent", "import_path": "harbor_adapter:NanoPyCodeAgent", "model_name": "test/model"}]}, + {"agents": []}, + {"agents": [{"name": "harbor_adapter:NanoPyCodeAgent", "model_name": "test/model", "kwargs": {"version": "old"}}]}, +]) +def test_conflicting_workflows_fail_before_a_run(change): + with pytest.raises(ValueError): + standard_config(config() | change) + + +def test_summary_does_not_turn_infrastructure_failure_into_zero(tmp_path): + failed = tmp_path / "failed" + failed.mkdir() + (failed / "result.json").write_text(json.dumps({"task_name": "qemu", "exception_info": {"exception_type": "AgentSetupTimeoutError"}, "verifier_result": None})) + scored = tmp_path / "scored" + scored.mkdir() + (scored / "result.json").write_text(json.dumps({"task_name": "qemu", "exception_info": None, "verifier_result": {"rewards": {"reward": 0.0}}})) + rows = summarize(tmp_path)["trials"] + assert rows[0]["exception_type"] == "AgentSetupTimeoutError" + assert rows[0]["rewards"] is None + assert rows[1]["status"] == "scored" + assert rows[1]["rewards"] == {"reward": 0.0} + + +def test_install_only_success_is_reported_without_a_score(tmp_path): + trial = tmp_path / "installed" + (trial / "agent").mkdir(parents=True) + (trial / "result.json").write_text(json.dumps({"task_name": "qemu"})) + (trial / "agent/bootstrap.json").write_text(json.dumps({"status": "ready", "cache_used": True})) + row, = summarize(tmp_path, install_only=True)["trials"] + assert row["status"] == "setup_passed" + assert row["rewards"] is None + assert row["bootstrap"]["cache_used"] is True + + +def test_exception_invalidates_a_partial_reward(tmp_path): + trial = tmp_path / "errored" + trial.mkdir() + (trial / "result.json").write_text(json.dumps({ + "exception_info": {"exception_type": "VerifierTimeoutError"}, + "verifier_result": {"rewards": {"reward": 0.0}}, + })) + row, = summarize(tmp_path)["trials"] + assert row["status"] == "execution_error" + assert row["rewards"] is None diff --git a/docs/changelogs/0.8.x.md b/docs/changelogs/0.8.x.md index 125ecaf..b92079a 100644 --- a/docs/changelogs/0.8.x.md +++ b/docs/changelogs/0.8.x.md @@ -5,6 +5,16 @@ All notable changes in the **0.8.x** release series are documented here. ## [Unreleased] ### Added +- Standardize Harbor runs through a recorded working-tree wheel, a shared + SHA256-verified APT cache, and task-revision-specific dependency preflight. + Add a QEMU Alpine SSH profile and an install-only check before model spend; + preserve setup failures separately from scored results and test the workflow + contracts in CI. +- Add an opt-in Harbor `RetryingVerifier` that retries a timed-out Linux test + script up to three total attempts at the task's native timeout, reusing the + existing agent output. Preserve attempt logs, clear stale rewards, and stop + timed-out test sessions before retrying. Completed scores, including zero, + and non-timeout errors are not retried. - Estimate model-call costs from token usage when a provider reports tokens but no per-request cost, starting with the DeepSeek official Anthropic-compatible endpoint. The estimate uses a static DeepSeek price diff --git a/src/nanopycodeagent/agent.py b/src/nanopycodeagent/agent.py index ced638e..d7773a9 100644 --- a/src/nanopycodeagent/agent.py +++ b/src/nanopycodeagent/agent.py @@ -841,7 +841,7 @@ def _reconcile_costs( *, max_seconds: float | None = None, ) -> list[JsonObject]: - """Append resolved OpenRouter costs without affecting the run outcome.""" + """Reconcile pending or estimated costs without affecting the run outcome.""" base_url = getattr(client, "base_url", "") credential = client.api_key or client.auth_token if not isinstance(credential, str) or not credential: @@ -867,7 +867,7 @@ def _reconcile_costs( not isinstance(generation_id, str) or generation_id in already_resolved or not isinstance(cost, dict) - or cost.get("status") != "pending" + or (cost.get("status") != "pending" and cost.get("kind") != "estimated") ): continue diagnostics: list[JsonObject] = [] diff --git a/src/nanopycodeagent/atif.py b/src/nanopycodeagent/atif.py index 9ae019b..9c5d9a1 100644 --- a/src/nanopycodeagent/atif.py +++ b/src/nanopycodeagent/atif.py @@ -271,6 +271,7 @@ def project_atif(entries: Sequence[JournalEntry]) -> JsonObject: if entry.type == "model.cost_resolved" } cost_states: list[tuple[str | None, Decimal | None]] = [] + estimated_costs: list[Decimal] = [] terminal: JournalEntry | None = None for entry in entries[1:]: payload = entry.payload @@ -390,6 +391,10 @@ def project_atif(entries: Sequence[JournalEntry]) -> JsonObject: metrics_extra = metrics.setdefault("extra", {}) assert isinstance(metrics_extra, dict) metrics_extra["cost_source"] = resolved_cost["source"] + if "kind" in resolved_cost: + metrics_extra["cost_kind"] = resolved_cost["kind"] + if resolved_cost.get("kind") == "estimated": + estimated_costs.append(amount) if generation_id is not None: metrics_extra["generation_id"] = generation_id if isinstance(cost, dict) or resolved_cost is not None: @@ -538,6 +543,13 @@ def project_atif(entries: Sequence[JournalEntry]) -> JsonObject: ] if missing: final_extra["missing_generation_ids"] = missing + if estimated_costs: + # Either the complete total or the known partial subtotal includes + # estimates. Keep their contribution separate from billed amounts. + final_extra = final_metrics.setdefault("extra", {}) + assert isinstance(final_extra, dict) + final_extra["cost_is_estimated"] = True + final_extra["estimated_cost_usd"] = float(sum(estimated_costs, Decimal(0))) return trajectory diff --git a/tests/test_agent_events.py b/tests/test_agent_events.py index b83eb58..8c4fe0b 100644 --- a/tests/test_agent_events.py +++ b/tests/test_agent_events.py @@ -7,6 +7,7 @@ import pytest from nanopycodeagent import agent, settings +from nanopycodeagent.atif import project_atif from nanopycodeagent.event_journal import EventJournal from helpers import ( @@ -188,6 +189,74 @@ def resolved(base_url, generation_id, credential, **kwargs): ] +@pytest.mark.parametrize( + ("generation_id", "provider_cost", "billed_cost", "expected_cost", "is_estimated"), + [ + (None, None, None, 0.0015, True), + ("gen-deepseek", None, "0.00072", 0.00072, False), + ("gen-deepseek", None, "0", 0.0, False), + ("gen-deepseek", None, None, 0.0015, True), + ("gen-deepseek", "0.0008", None, 0.0008, False), + ("gen-deepseek", "0", None, 0.0, False), + ], +) +def test_deepseek_estimate_preserves_billing_reconciliation( + monkeypatch, generation_id, provider_cost, billed_cost, expected_cost, is_estimated +): + usage = {"input_tokens": 1000, "output_tokens": 1000} + if provider_cost is not None: + usage["cost"] = provider_cost + reply = FakeStream( + [text_block("done")], + model="deepseek-flash", + usage=SimpleNamespace(**usage), + response_headers={"x-generation-id": generation_id} if generation_id else {}, + ) + patch_client(monkeypatch, FakeClient(FakeMessages([reply]))) + lookups = [] + + def resolve(base_url, requested_id, credential, **kwargs): + lookups.append(requested_id) + if billed_cost is None: + return None + return { + "generation_id": requested_id, + "amount": billed_cost, + "currency": "USD", + "source": "provider_generation.total_cost", + } + + monkeypatch.setattr(agent, "resolve_generation_cost", resolve) + + assert agent.run_headless("fix it") == 0 + + should_reconcile = generation_id is not None and provider_cost is None + assert lookups == ([generation_id] if should_reconcile else []) + entries = EventJournal.replay(_only_journal_path()) + if should_reconcile: + assert entries[-1].payload["cost_reconciliation"][0]["status"] == ( + "resolved" if billed_cost is not None else "unresolved" + ) + trajectory = project_atif(entries) + metrics = trajectory["final_metrics"] + step_metrics = next( + step["metrics"] for step in trajectory["steps"] if step["source"] == "agent" + ) + assert step_metrics["cost_usd"] == expected_cost + assert metrics["total_cost_usd"] == expected_cost + extra = metrics.get("extra", {}) + assert extra.get("cost_is_estimated", False) is is_estimated + if is_estimated: + assert step_metrics["extra"]["cost_kind"] == "estimated" + assert extra["estimated_cost_usd"] == expected_cost + else: + assert step_metrics["extra"]["cost_source"] == ( + "provider_response.usage.cost" if provider_cost is not None + else "provider_generation.total_cost" + ) + assert "estimated_cost_usd" not in extra + + def test_unresolved_cost_diagnostics_are_persisted_on_run_terminal(monkeypatch): reply = FakeStream( [text_block("done")], diff --git a/tests/test_atif.py b/tests/test_atif.py index 25bbdcc..901ad59 100644 --- a/tests/test_atif.py +++ b/tests/test_atif.py @@ -2,6 +2,7 @@ import json import stat +from dataclasses import replace from pathlib import Path import pytest @@ -223,6 +224,48 @@ def test_resolved_and_missing_costs_project_as_partial_metrics(tmp_path): assert trajectory["final_metrics"]["total_cost_usd"] == 0.00072 +@pytest.mark.parametrize("estimated_amount", ["0", "0.0015"]) +@pytest.mark.parametrize("other_kind", ["estimated", "provider_reported", "pending"]) +def test_estimated_cost_summary_preserves_mixed_and_partial_costs( + tmp_path, estimated_amount, other_kind +): + entries = _journal_entries(tmp_path) + completed = next(entry for entry in entries if entry.type == "model.completed") + completed.payload["generation_id"] = None + completed.payload["cost"] = { + "status": "resolved", "amount": estimated_amount, "currency": "USD", + "source": "token_estimate.deepseek", "kind": "estimated", + } + other_cost = {"status": "pending", "source": "provider_generation"} + if other_kind != "pending": + other_cost = { + "status": "resolved", "amount": "0.002", "currency": "USD", + "source": "token_estimate.deepseek" if other_kind == "estimated" + else "provider_response.usage.cost", + "kind": other_kind, + } + entries.insert(-1, replace(completed, payload=completed.payload | { + "model_call_id": "model-2", "generation_id": "generation-2", "cost": other_cost, + })) + + trajectory = project_atif(entries) + metrics = trajectory["final_metrics"] + extra = metrics["extra"] + assert extra["cost_is_estimated"] is True + assert extra["estimated_cost_usd"] == pytest.approx( + float(estimated_amount) + (0.002 if other_kind == "estimated" else 0) + ) + assert trajectory["steps"][1]["metrics"]["extra"]["cost_kind"] == "estimated" + if other_kind == "pending": + assert "total_cost_usd" not in metrics + assert extra["known_cost_usd"] == float(estimated_amount) + assert extra["cost_is_partial"] is True + assert extra["missing_generation_ids"] == ["generation-2"] + else: + assert metrics["total_cost_usd"] == pytest.approx(float(estimated_amount) + 0.002) + assert "cost_is_partial" not in extra + + def test_cost_reconciliation_diagnostics_project_to_terminal_extra(tmp_path): entries = _journal_entries(tmp_path) entries[-1].payload["cost_reconciliation"] = [ From bfc12157cff610e44caba3872bc2defd86e72557 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Mon, 5 Oct 2026 00:07:42 -0700 Subject: [PATCH 5/6] fix(harbor): enforce preflight and preserve attempt logs --- .../harbor/src/harbor_adapter/__main__.py | 2 +- .../harbor/src/harbor_adapter/bootstrap.py | 10 ++- .../harbor/src/harbor_adapter/verifier.py | 13 +++- benchmarks/harbor/tests/test_bootstrap.py | 43 ++++++++++++ benchmarks/harbor/tests/test_verifier.py | 69 ++++++++++++++++++- benchmarks/harbor/tests/test_workflow.py | 15 ++++ 6 files changed, 145 insertions(+), 7 deletions(-) diff --git a/benchmarks/harbor/src/harbor_adapter/__main__.py b/benchmarks/harbor/src/harbor_adapter/__main__.py index 2b07ce6..d41cf39 100644 --- a/benchmarks/harbor/src/harbor_adapter/__main__.py +++ b/benchmarks/harbor/src/harbor_adapter/__main__.py @@ -42,7 +42,7 @@ def standard_config(raw: dict) -> JobConfig: raise ValueError( "Use native per-attempt timeouts and verifier_timeout_multiplier=4" ) - if config.verifier.disable or config.source_jobs: + if (config.verifier.disable and not config.install_only) or config.source_jobs: raise ValueError("The standard workflow runs fresh trials with verification enabled") if not config.tasks and not config.datasets: raise ValueError("Select at least one task or dataset") diff --git a/benchmarks/harbor/src/harbor_adapter/bootstrap.py b/benchmarks/harbor/src/harbor_adapter/bootstrap.py index 845f6ce..412315c 100644 --- a/benchmarks/harbor/src/harbor_adapter/bootstrap.py +++ b/benchmarks/harbor/src/harbor_adapter/bootstrap.py @@ -165,15 +165,19 @@ def verifier_profile( logs_dir: Path, environment_name: str | None, ) -> VerifierProfile | None: known_tasks = {name for name, _ in VERIFIER_PROFILES} + known_environment = environment_name in {name.split("/")[-1] for name in known_tasks} config_path = logs_dir.parent / "config.json" if not config_path.exists(): - if environment_name in {name.split("/")[-1] for name in known_tasks}: + if known_environment: raise ValueError("Verifier preflight requires Harbor's pinned trial config") return None task = json.loads(config_path.read_text())["task"] profile = VERIFIER_PROFILES.get((task.get("name"), task.get("ref"))) - if profile is None and task.get("name") in known_tasks: + # Local and Git datasets omit package name/ref. The environment identifies + # a known task, but cannot establish its reviewed revision by itself. + if profile is None and (task.get("name") in known_tasks or known_environment): + name = task.get("name") if task.get("name") in known_tasks else environment_name raise ValueError( - f"Unknown {task['name']} revision; review its verifier dependency profile" + f"Unknown {name} revision; review its verifier dependency profile" ) return profile diff --git a/benchmarks/harbor/src/harbor_adapter/verifier.py b/benchmarks/harbor/src/harbor_adapter/verifier.py index 930310b..7fd6e3f 100644 --- a/benchmarks/harbor/src/harbor_adapter/verifier.py +++ b/benchmarks/harbor/src/harbor_adapter/verifier.py @@ -125,7 +125,7 @@ async def verify(self): target_dir=self.trial_paths.verifier_dir, include=self.include_logs or None, exclude=self.exclude_logs or None, - protect=["reward.txt", "reward.json", "attempts/**", "retry-summary.json"], + protect=["reward.txt", "reward.json", "retry-summary.json"], ) else: download = environment.download_dir( @@ -137,3 +137,14 @@ async def verify(self): # Collection failure must not hide the original timeout, # cancellation, or reward parsing error. self.logger.exception("Failed to collect verifier attempt logs") + if self.include_logs or self.exclude_logs: + # Harbor protects exact file paths only. Download every + # archived attempt separately, regardless of log filters + # or failure to collect the current attempt's logs. + try: + await asyncio.wait_for(environment.download_dir( + source_dir=str(paths.verifier_dir / "attempts"), + target_dir=self.trial_paths.verifier_dir / "attempts", + ), timeout=30) + except Exception: + self.logger.exception("Failed to collect archived verifier attempts") diff --git a/benchmarks/harbor/tests/test_bootstrap.py b/benchmarks/harbor/tests/test_bootstrap.py index b11bec1..7c5ff81 100644 --- a/benchmarks/harbor/tests/test_bootstrap.py +++ b/benchmarks/harbor/tests/test_bootstrap.py @@ -6,8 +6,10 @@ import json from pathlib import Path from types import SimpleNamespace +from unittest.mock import AsyncMock import pytest +from harbor.models.trial.config import TrialConfig from harbor_adapter import NanoPyCodeAgent from harbor_adapter.bootstrap import AptCache, Package, QEMU_REF, QEMU_TASK, parse_apt_plan, verifier_profile @@ -133,6 +135,47 @@ def test_profile_is_scoped_to_reviewed_task_revision(tmp_path): verifier_profile(logs_dir, "qemu-alpine-ssh") +@pytest.mark.parametrize("task", [ + {"path": "/tasks/qemu-alpine-ssh"}, + {"path": "/dataset/qemu-alpine-ssh", "source": "local-dataset"}, + { + "path": "tasks/qemu-alpine-ssh", "source": "git-dataset", + "git_url": "https://example.com/tasks.git", "git_commit_id": "a" * 40, + }, +], ids=["local-path", "local-dataset", "git-dataset"]) +def test_unresolved_qemu_revision_stops_setup_before_install(tmp_path, monkeypatch, task): + logs_dir = tmp_path / "agent" + logs_dir.mkdir() + (tmp_path / "config.json").write_text(TrialConfig(task=task).model_dump_json()) + adapter = NanoPyCodeAgent(logs_dir=logs_dir) + dependencies = AsyncMock() + install = AsyncMock() + monkeypatch.setattr(adapter, "_ensure_system_dependencies", dependencies) + monkeypatch.setattr(adapter, "_install_agent", install) + + with pytest.raises(ValueError, match="Unknown .*qemu-alpine-ssh revision"): + asyncio.run(adapter.install(SimpleNamespace(environment_name="qemu-alpine-ssh"))) + + dependencies.assert_not_awaited() + install.assert_not_awaited() + report = json.loads((logs_dir / "bootstrap.json").read_text()) + assert report["status"] == "failed" + assert report["stage"] == "profile" + + +def test_known_environment_requires_trial_config(tmp_path): + with pytest.raises(ValueError, match="pinned trial config"): + verifier_profile(tmp_path / "agent", "qemu-alpine-ssh") + + +@pytest.mark.parametrize("has_config", [False, True]) +def test_unregistered_task_does_not_require_a_profile(tmp_path, has_config): + if has_config: + config = TrialConfig(task={"path": "/tasks/other-task"}) + (tmp_path / "config.json").write_text(config.model_dump_json()) + assert verifier_profile(tmp_path / "agent", "other-task") is None + + def test_preflight_failure_stops_setup_and_records_the_failing_stage(tmp_path, monkeypatch): logs_dir = tmp_path / "agent" write_task(logs_dir) diff --git a/benchmarks/harbor/tests/test_verifier.py b/benchmarks/harbor/tests/test_verifier.py index 5ff0372..075c677 100644 --- a/benchmarks/harbor/tests/test_verifier.py +++ b/benchmarks/harbor/tests/test_verifier.py @@ -18,6 +18,7 @@ from harbor.trial.single_step import SingleStepTrial from harbor.trial.trial import Trial from harbor.trial.errors import VerifierTimeoutError +from harbor.utils.path_filter import filter_paths_by_patterns from harbor.verifier.factory import VerifierFactory from harbor.verifier.verifier import RewardFileNotFoundError, VerifierOutputParseError @@ -44,7 +45,19 @@ async def upload_dir(self, **kwargs): async def download_dir(self, *, source_dir, target_dir): self.downloads += 1 - shutil.copytree(self.logs, target_dir, dirs_exist_ok=True) + source = self.logs / Path(source_dir).relative_to("/logs/verifier") + shutil.copytree(source, target_dir, dirs_exist_ok=True) + + async def download_dir_filtered(self, *, source_dir, target_dir, include=None, exclude=None, protect=None): + source = self.logs / Path(source_dir).relative_to("/logs/verifier") + paths = [path.relative_to(source).as_posix() for path in source.rglob("*") if path.is_file()] + selected = set(filter_paths_by_patterns(paths, include=include, exclude=exclude)) + # Harbor's protected entries are exact relative paths, never globs. + selected.update(set(protect or []) & set(paths)) + for path in selected: + target = Path(target_dir) / path + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source / path, target) async def exec(self, *, command, env=None, **kwargs): if command.startswith("chmod +x"): @@ -68,7 +81,7 @@ async def exec(self, *, command, env=None, **kwargs): return ExecResult(return_code=process.returncode, stdout=stdout.decode(), stderr=stderr.decode()) -def make_verifier(tmp_path, script, *, mounted=True, max_attempts=3): +def make_verifier(tmp_path, script, *, mounted=True, max_attempts=3, include_logs=None, exclude_logs=None): task_dir = tmp_path / "task" (task_dir / "tests").mkdir(parents=True) (task_dir / "instruction.md").write_text("Write the solution.") @@ -82,6 +95,7 @@ def make_verifier(tmp_path, script, *, mounted=True, max_attempts=3): verifier=VerifierConfig( import_path="harbor_adapter:RetryingVerifier", kwargs={"max_attempts": max_attempts}, + include_logs=include_logs or [], exclude_logs=exclude_logs or [], ), ) paths.config_path.write_text(config.model_dump_json()) @@ -165,6 +179,44 @@ def test_three_timeouts_leave_no_canonical_reward_and_keep_all_logs(tmp_path, mo assert environment.downloads == (0 if mounted else 1) +@pytest.mark.parametrize("filters", [ + {"include_logs": ["test-stdout.txt"]}, + {"exclude_logs": ["attempts/*", "retry-summary.json", "reward.txt", "details/*"]}, +], ids=["include-current-stdout", "exclude-archives"]) +@pytest.mark.parametrize("exhausted", [False, True], ids=["completed", "exhausted"]) +def test_log_filters_preserve_every_attempt_on_non_mounted_environments(tmp_path, filters, exhausted): + script = ''' +count=$(cat "$STATE" 2>/dev/null || echo 0) +count=$((count + 1)) +echo "$count" > "$STATE" +echo "attempt $count" +mkdir -p "$LOGS/details" +echo "detail $count" > "$LOGS/details/check.txt" +echo 1 > "$LOGS/reward.txt" +''' + script += 'sleep 10\n' if exhausted else 'if [ "$count" -lt 3 ]; then sleep 10; fi\n' + verifier, environment, _ = make_verifier(tmp_path, script, mounted=False, **filters) + if exhausted: + with pytest.raises(TimeoutError, match="3 attempts"): + asyncio.run(verifier.verify()) + else: + assert asyncio.run(verifier.verify()).rewards == {"reward": 1} + + # Only downloaded files survive environment teardown. + shutil.rmtree(environment.logs) + logs = verifier.trial_paths.verifier_dir + assert [item["status"] for item in summary(verifier)["attempts"]] == [ + "timeout", "timeout", "timeout" if exhausted else "completed", + ] + for attempt in (1, 2, 3): + archive = logs / "attempts" / str(attempt) + assert (archive / "test-stdout.txt").read_text() == f"attempt {attempt}\n" + assert (archive / "details/check.txt").read_text() == f"detail {attempt}\n" + assert (archive / "reward.txt").read_text() == "1\n" + assert verifier.trial_paths.reward_text_path.exists() is (not exhausted) + assert not (logs / "details/check.txt").exists() + + @pytest.mark.parametrize(("script", "error"), [ ("exit 124\n", RewardFileNotFoundError), ('echo invalid > "$LOGS/reward.txt"\n', VerifierOutputParseError), @@ -252,6 +304,19 @@ def test_log_download_failure_preserves_harbor_timeout_error(tmp_path, caplog): environment.download_dir.assert_awaited_once() +def test_filtered_download_failure_still_collects_attempt_archives(tmp_path, caplog): + verifier, environment, _ = make_verifier( + tmp_path, 'echo started\nsleep 10\n', mounted=False, max_attempts=1, + include_logs=["test-stdout.txt"], + ) + environment.download_dir_filtered = AsyncMock(side_effect=OSError("download failed")) + with pytest.raises(TimeoutError, match="1 attempts"): + asyncio.run(verifier.verify()) + assert "Failed to collect verifier attempt logs" in caplog.text + archive = verifier.trial_paths.verifier_dir / "attempts/1/test-stdout.txt" + assert archive.read_text() == "started\n" + + def test_outer_timeout_cap_is_applied_before_multiplier(tmp_path): verifier, _, config = make_verifier(tmp_path, 'echo 1 > "$LOGS/reward.txt"\n') config.verifier.max_timeout_sec = 0.3 diff --git a/benchmarks/harbor/tests/test_workflow.py b/benchmarks/harbor/tests/test_workflow.py index 94e361f..8d61ad0 100644 --- a/benchmarks/harbor/tests/test_workflow.py +++ b/benchmarks/harbor/tests/test_workflow.py @@ -23,6 +23,21 @@ def test_standard_policy_enables_native_verifier_retries_without_agent_retries() assert result.verifier_timeout_multiplier == 4 +def test_install_only_json_config_is_accepted_and_round_trips(): + result = standard_config(config() | {"install_only": True}) + assert result.install_only is True + assert result.verifier.disable is True + assert standard_config(json.loads(result.model_dump_json())) == result + + +def test_full_runs_still_require_verification(): + raw = config() | { + "verifier": {"import_path": "harbor_adapter:RetryingVerifier", "disable": True}, + } + with pytest.raises(ValueError, match="verification enabled"): + standard_config(raw) + + @pytest.mark.parametrize("change", [ {"retry": {"max_retries": 1}}, {"verifier_timeout_multiplier": 1}, From 3c495e23b3d007b62e5af6faa4ffcc41ad829964 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Mon, 5 Oct 2026 00:08:21 -0700 Subject: [PATCH 6/6] docs: complete changelog for benchmark reliability --- docs/changelogs/0.8.x.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelogs/0.8.x.md b/docs/changelogs/0.8.x.md index b92079a..83c73b4 100644 --- a/docs/changelogs/0.8.x.md +++ b/docs/changelogs/0.8.x.md @@ -42,6 +42,11 @@ All notable changes in the **0.8.x** release series are documented here. supported. ### Fixed +- Preserve estimated-cost classification and subtotals in ATIF and Harbor + summaries, while allowing generation billing lookups to replace estimates. +- Reject unresolved QEMU task revisions selected through local paths or + datasets, accept install-only mode from JSON configuration, and preserve + archived verifier attempts when Harbor filters downloaded logs. - Retry the Harbor adapter's system-dependency installation a bounded number of times when the package index returns a transient HTTP 404, instead of failing agent setup immediately. Non-404 failures and the final attempt still