From e7ee08de534b4d359eaf773c5ac4baaae1065fba Mon Sep 17 00:00:00 2001 From: david6666666 <530634352@qq.com> Date: Mon, 26 Oct 2026 08:33:28 +0000 Subject: [PATCH] feat: add bounded offline harness improvement policy Agent-assisted-by: Codex (model and effort unavailable) Signed-off-by: david6666666 <530634352@qq.com> --- README.md | 21 ++ docs/architecture.md | 19 ++ docs/examples/harness-improvement/README.md | 42 +++ .../harness-improvement/bounded-report.json | 131 ++++++++++ .../harness-improvement/offline-summary.json | 65 +++++ docs/harness-improvement.md | 151 +++++++++++ docs/operations.md | 1 + docs/skill-eval-comparison.md | 6 + src/inferrsi/harness/improvement.py | 115 +++++++++ src/inferrsi/harness/skill_evaluation.py | 70 ++++- .../fixtures/skill_eval_improvement_client.py | 19 ++ tests/fixtures/skill_eval_improvement_demo.py | 75 ++++++ .../skill_eval_improvement_support.py | 79 ++++++ tests/test_skill_improvement.py | 209 +++++++++++++++ tools/skill_eval_compare.py | 31 ++- tools/skill_eval_improve.py | 241 ++++++++++++++++++ tools/skill_eval_pilot.py | 4 +- 17 files changed, 1263 insertions(+), 16 deletions(-) create mode 100644 docs/examples/harness-improvement/README.md create mode 100644 docs/examples/harness-improvement/bounded-report.json create mode 100644 docs/examples/harness-improvement/offline-summary.json create mode 100644 docs/harness-improvement.md create mode 100644 src/inferrsi/harness/improvement.py create mode 100644 tests/fixtures/skill_eval_improvement_client.py create mode 100644 tests/fixtures/skill_eval_improvement_demo.py create mode 100644 tests/fixtures/skill_eval_improvement_support.py create mode 100644 tests/test_skill_improvement.py create mode 100644 tools/skill_eval_improve.py diff --git a/README.md b/README.md index 8db242d06..d7d5fb1b8 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,22 @@ Knowledge retrieval is read-only; candidate storage is optional and separately authorized. Automatic search and automatic knowledge promotion remain planned. Missing or incomplete evidence stays `INCONCLUSIVE`. +An optional [offline harness-improvement path](docs/harness-improvement.md) +now checks caller-prepared skill edits using the existing Skill Eval runner: + +```mermaid +flowchart LR + A["H0 + INV / DOM skill snapshots"] --> B["Matched development probes"] + B --> C["Quality-first, cost-aware choice or H0"] + C --> D["Mandatory rerun"] + D -->|"eligible candidate"| E["One-shot holdout gate"] + E --> F["Bounded candidate or retain H0"] + D -->|"H0 / invalid evidence"| F +``` + +This path is verified with fixed-output fixtures. Real-model automatic editing, +cross-task reuse and measured model savings remain unverified. + ## Choose your starting path | You want to… | Start at | @@ -137,6 +153,11 @@ Ready to measure your own model? [Connect your benchmark and quality checks →] The implemented [Harness × Model × Infra boundary](docs/architecture.md) keeps new campaigns out of the common AB/BA scheduler, process cleanup and evaluator. +The [bounded skill policy](docs/harness-improvement.md#implemented-architecture) +sits above Skill Eval, reuses its comparison runner and independent verifier, +and records development experience separately from executor-facing skills. +Engine performance claims continue through the engine AB/BA lane. + ![InferRSI system architecture: inputs, experiment loop and evidence](docs/assets/system-architecture-en.svg) [English SVG](docs/assets/system-architecture-en.svg) · diff --git a/docs/architecture.md b/docs/architecture.md index 759177364..7a0194bf9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -83,6 +83,8 @@ src/inferrsi/ runner.py One AB/BA loop; execute/snapshot adapter hooks evaluation.py Performance/quality/source-integrity adjudication reporting.py HTML evidence report + skill_evaluation.py Skill trials, usage summaries and shared trial budget + improvement.py Offline quality/cost selection policy; no model execution scenarios/ external.py External benchmark/validator commands; metric-reader hook media.py Warmup seeds, first-response retention, sample/artifact groups @@ -116,6 +118,23 @@ examples/ Current Qwen runner, image validator and v4 adapter tests/fixtures/legacy/ Frozen historical validation dependencies for regressions ``` +### Bounded skill improvement + +[`tools/skill_eval_improve.py`](../tools/skill_eval_improve.py) composes the existing +Skill Eval comparison tool for caller-prepared H0/INV/DOM snapshots. It freezes +the supported audit inputs and policy, shares one outer budget across probes, +mandatory rerun and one-shot holdout, and revalidates retained evidence when +reporting. The Harness policy module owns quality gates and cost selection; +the existing comparison runner and Infra executor still own trials and cleanup. +See the [implemented flow and offline limits](harness-improvement.md). + +This is a separate optional skill-evaluation path. It neither replaces engine +AB/BA adjudication nor automatically writes/promotes knowledge. The initial +executor accepts only explicitly supplied trusted offline fixtures; automatic +model edits, external sandbox qualification and cross-task reuse remain future +work. Shared total-token and command-count summaries also serve the original +pilot, configured comparison and PR integration consumers. + ## Where to extend | Add | Main extension locations | Required qualification | diff --git a/docs/examples/harness-improvement/README.md b/docs/examples/harness-improvement/README.md new file mode 100644 index 000000000..ad7faa9ee --- /dev/null +++ b/docs/examples/harness-improvement/README.md @@ -0,0 +1,42 @@ +# Recorded offline controller example + +These are fixed-output software controls for +[issue #93](https://github.com/InferMatrixLab/InferRSI/issues/93), generated with +the adjacent implementation and `tests.fixtures.skill_eval_improvement_demo`. +Python 3.12.3 and Git 2.47.3 were used on Linux. No real model was invoked. +Full raw trial artifacts are produced locally by the reproduction command; +these public JSON files retain the aggregate report without private paths. + +| Control | Actual subprocess calls | Scripted total tokens | Scripted held-out candidate checks | Regressions | +| --- | ---: | ---: | ---: | ---: | +| Paired H0/H0 | 12 | 1,440 | 6/6 | 0 | +| Repeated H0, same call budget as the loop | 48 | 5,760 | 24/24 | 0 | +| Bounded INV/DOM → rerun → holdout | 48 | 4,200 | 6/6 | 0 | + +The cheaper DOM fixture costs 240 scripted tokens over six development trials +but has three regressions; the controller rejects it. INV costs 360 versus its +paired H0's 720, passes the mandatory rerun and then the held-out checks. The +full loop records 4,200, not just the selected candidate's 360. These values +are deliberately programmed inputs to test quality-first selection and complete +accounting. **They do not show a measured percentage saving or success-rate +improvement for InferRSI.** The repeated-H0 control matches call count, not billed +cost or a hard token budget, and has no best-of-N answer selection. + +The [summary](offline-summary.json) separates preparation and local pipeline +wall time. A single local timing observation is not a speedup benchmark; these +fixtures were also run alongside repository tests. The +[bounded report](bounded-report.json) retains phase costs, selection reasons, +rerun/holdout assessments and unknown proposal/monetary cost. Cached-input tokens +are recorded separately and are not added to total tokens twice. + +Reproduce from the repository checkout with development dependencies: + +```bash +python -m tests.fixtures.skill_eval_improvement_demo --output +``` + +The generated JSON includes retained-plan/report digests for that invocation. +Temporary Git commit identities, file digests and wall times can differ between +runs. The scripted call/token/outcome values above are reproducible. Model +efficacy, complete authoring-plus-execution cost and engine benefits remain +`INCONCLUSIVE`; this example is not a campaign `PASS`. diff --git a/docs/examples/harness-improvement/bounded-report.json b/docs/examples/harness-improvement/bounded-report.json new file mode 100644 index 000000000..111de0ef0 --- /dev/null +++ b/docs/examples/harness-improvement/bounded-report.json @@ -0,0 +1,131 @@ +{ + "plan_sha256": "81e4d4e425ea4eb58ea6dd7f145e4fc17df3bc37197cd7d3e1e430cfd6e82031", + "selection": { + "selected": "INV", + "reason": "quality_then_cost", + "assessments": { + "INV": { + "eligible": true, + "reason": null, + "success_rate": 1.0, + "baseline_success_rate": 1.0, + "total_tokens": 360, + "baseline_total_tokens": 720, + "regressions": 0 + }, + "DOM": { + "eligible": false, + "reason": "quality_floor", + "success_rate": 0.5, + "baseline_success_rate": 1.0, + "total_tokens": 240, + "baseline_total_tokens": 720, + "regressions": 3 + } + }, + "pareto": [ + "INV" + ], + "scores": { + "INV": 0.0 + }, + "plan_sha256": "81e4d4e425ea4eb58ea6dd7f145e4fc17df3bc37197cd7d3e1e430cfd6e82031" + }, + "retained_selection_consistent": true, + "rerun": { + "eligible": true, + "reason": null, + "success_rate": 1.0, + "baseline_success_rate": 1.0, + "total_tokens": 360, + "baseline_total_tokens": 720, + "regressions": 0 + }, + "holdout": { + "eligible": true, + "reason": null, + "success_rate": 1.0, + "baseline_success_rate": 1.0, + "total_tokens": 360, + "baseline_total_tokens": 720, + "regressions": 0 + }, + "accepted": true, + "effective_candidate": "INV", + "phase_usage": { + "probe-INV": { + "calls": 12, + "input_tokens": 960, + "output_tokens": 120, + "cached_input_tokens": 240, + "total_tokens": 1080, + "command_calls": 24, + "elapsed_seconds": 0.20449978415854275 + }, + "probe-DOM": { + "calls": 12, + "input_tokens": 840, + "output_tokens": 120, + "cached_input_tokens": 240, + "total_tokens": 960, + "command_calls": 24, + "elapsed_seconds": 0.1887140409089625 + }, + "rerun-INV": { + "calls": 12, + "input_tokens": 960, + "output_tokens": 120, + "cached_input_tokens": 240, + "total_tokens": 1080, + "command_calls": 24, + "elapsed_seconds": 0.1887151626870036 + }, + "holdout-INV": { + "calls": 12, + "input_tokens": 960, + "output_tokens": 120, + "cached_input_tokens": 240, + "total_tokens": 1080, + "command_calls": 24, + "elapsed_seconds": 0.18848644592799246 + } + }, + "observed_execution": { + "calls": 48, + "input_tokens": 3720, + "output_tokens": 480, + "cached_input_tokens": 960, + "total_tokens": 4200, + "command_calls": 96, + "elapsed_seconds": 2.1600983038078994, + "cost": null + }, + "budget": { + "calls": 48, + "observed_tokens": 4200, + "usage_complete": true, + "stop_reason": "call_budget" + }, + "proposal_cost": null, + "proposal_cost_note": "Caller-prepared snapshots; authoring cost is unknown and excluded.", + "selected_execution": { + "trials": 6, + "verdicts": { + "PASS": 6, + "FAIL": 0, + "INCONCLUSIVE": 0 + }, + "evaluation_errors": 0, + "elapsed_seconds": 0.09431080287322402, + "input_tokens": 300, + "output_tokens": 60, + "cached_input_tokens": 120, + "total_tokens": 360, + "command_calls": 12, + "cost": null, + "expected_trials": 6, + "missing_trials": 0 + }, + "promotion": "not_evaluated", + "claim": "Offline control-flow evidence only. Fixture tokens are scripted, not billed. No measured model efficacy, end-to-end savings, sandbox isolation or engine speedup." +} diff --git a/docs/examples/harness-improvement/offline-summary.json b/docs/examples/harness-improvement/offline-summary.json new file mode 100644 index 000000000..707f43f1f --- /dev/null +++ b/docs/examples/harness-improvement/offline-summary.json @@ -0,0 +1,65 @@ +{ + "kind": "offline_fixture_control", + "rows": [ + { + "condition": "H0", + "preparation_seconds": 0.05566692119464278, + "execution_seconds": 0.5605955601204187, + "calls": 12, + "scripted_total_tokens": 1440, + "scripted_command_calls": 24, + "holdout_candidate_trials": 6, + "holdout_candidate_verdicts": { + "PASS": 6, + "FAIL": 0, + "INCONCLUSIVE": 0 + }, + "holdout_regressions": 0, + "effective_candidate": "H0", + "plan_sha256": "bd0d7c0062dfe8bf244b46cb8b7423e4db65d08c651f5207a9dbf07f446f79ad", + "report_sha256": "9c1c68988dc512a3bc95b0004de5ba7d0bc3f9dfd689ca46675592a1a8f00fb2" + }, + { + "condition": "repeated-H0", + "preparation_seconds": 0.04197078011929989, + "execution_seconds": 1.9998727978672832, + "calls": 48, + "scripted_total_tokens": 5760, + "scripted_command_calls": 96, + "holdout_candidate_trials": 24, + "holdout_candidate_verdicts": { + "PASS": 24, + "FAIL": 0, + "INCONCLUSIVE": 0 + }, + "holdout_regressions": 0, + "effective_candidate": "H0", + "plan_sha256": "6ebec22ce01f12d61ec07c00711f3c190206b0830d92345adbdceb4a42789f21", + "report_sha256": "ba0866d1d2f145117eb122a1c844c2ecc5ee0eb6e3b79e3231e3263a8a2f0c2a" + }, + { + "condition": "bounded", + "preparation_seconds": 0.1715290721040219, + "execution_seconds": 2.1994067209307104, + "calls": 48, + "scripted_total_tokens": 4200, + "scripted_command_calls": 96, + "holdout_candidate_trials": 6, + "holdout_candidate_verdicts": { + "PASS": 6, + "FAIL": 0, + "INCONCLUSIVE": 0 + }, + "holdout_regressions": 0, + "effective_candidate": "INV", + "plan_sha256": "81e4d4e425ea4eb58ea6dd7f145e4fc17df3bc37197cd7d3e1e430cfd6e82031", + "report_sha256": "b0e3d7f1d17708a737687b6aca07838f42a45919071343e9e74634812b3bb04b" + } + ], + "scope": "Actual subprocess calls and pipeline wall time; scripted answers and tokens. H0 is a paired H0/H0 control, repeated-H0 has the bounded loop's 48-call budget. No best-of-N answer selection. Different repeat counts are not efficacy estimates.", + "model_calls": 0, + "monetary_cost": null, + "proposal_authoring_cost": null, + "efficacy_verdict": "INCONCLUSIVE", + "engine_performance_verdict": "INCONCLUSIVE" +} diff --git a/docs/harness-improvement.md b/docs/harness-improvement.md new file mode 100644 index 000000000..a10dfc975 --- /dev/null +++ b/docs/harness-improvement.md @@ -0,0 +1,151 @@ +# Bounded harness improvement (offline first slice) + +The policy from [issue #93](https://github.com/InferMatrixLab/InferRSI/issues/93) +now has an offline controller above the existing Skill Eval comparison tool. +It evaluates caller-prepared INV/DOM Markdown edits, chooses one whole candidate +or H0, reruns that choice, then applies a single held-out acceptance gate. +This is **verifier-guided search**, a deliberate adaptation of Harness². +It is not autonomous proposal generation or a demonstrated model improvement. + +## What the two approaches contribute + +| Source | Mechanism used here | Outside this slice | +| --- | --- | --- | +| [Harness² v1](https://arxiv.org/html/2610.14224v1) | A task-local reference, general INV and domain DOM alternatives, matched probes, successor execution and separate improvement experience | Model-generated edits, arbitrary composition, recursive rounds and cross-task library reuse | +| [HarnessPro v2](https://arxiv.org/html/2610.26770v2) | Quality and regression gates before Pareto filtering; normalized weighted squared-distance selection; execution cost separate from search overhead | Generated tests, executable hooks, offline population search and Lite/Medium/Heavy deployment families | + +Neither paper's reported gains establish InferRSI gains. The existing independent +audit grader remains authoritative. The default task is performance-report +auditing, not general code optimization. PR integration is already delivered +by [#118](https://github.com/InferMatrixLab/InferRSI/pull/118); the older issue +comments' draft-only description is historical. + +## Implemented architecture + +```mermaid +flowchart TB + Inputs["Caller-prepared H0 / INV / DOM Git snapshots"] --> Freeze + Freeze["Freeze Markdown allowlist, split, grader, model, policy and total limits"] + Freeze --> Probes["Matched development probes via existing Skill Eval"] + Probes --> Select["Quality floor + paired regressions → Pareto + weighted cost choice"] + Select --> Choice["Freeze INV, DOM or H0; no automatic edit merging"] + Choice --> Rerun["Mandatory development rerun through the same runner"] + Rerun --> Benefit{"Quality and benefit retained?"} + Benefit -->|"yes: selected candidate"| Holdout["One-shot frozen holdout"] + Benefit -->|"no / missing / error"| H0["Retain H0"] + Holdout --> Gate{"Independent quality, regression and benefit gates"} + Gate -->|accept| Candidate["Bounded candidate; no promotion"] + Gate -->|reject| H0 + Select --> Experience["Development-only improvement experience"] + Rerun --> Experience + Budget["Outer call/deadline budget and observed-token admission stop"] -.-> Probes + Budget -.-> Rerun + Budget -.-> Holdout + Engine["Engine RunSpec"] --> ABBA["Existing engine AB / BA"] --> Evidence["Separate engine evidence"] +``` + +No holdout feedback reaches selection or the experience artifact. H0 is an +explicit option in the Pareto set: its reference uses the pooled, equally sized +H0 probe executions, while each candidate still must pass its own paired H0 +quality/regression checks. The weights and absolute minimum success rate come +from the frozen config. Missing/inconclusive evidence or unknown input/output +usage keeps H0. Flat normalized dimensions contribute zero; ties prefer H0, +then the candidate ID. A same-quality successor must remain cheaper on both the +development rerun and holdout. A quality gain may trade off against more tokens +according to the predeclared selection policy; no monetary price is inferred. + +Composition in this slice selects INV, DOM or H0 as a complete snapshot. It +does not union two edits, so conflicting edits cannot silently be combined. +Source snapshots are materialized through the existing Git/KnowledgeReader +adapter; no second runner or Store-to-execution shortcut is introduced. + +## Interfaces and reproduction + +Requires development dependencies and Git 2.45 or newer, as does +[configured comparison](skill-eval-comparison.md). + +```bash +python -m tests.fixtures.skill_eval_improvement_demo --output +python -m tools.skill_eval_improve report --output /bounded/run +pytest tests/test_skill_improvement.py +``` + +The demo creates a tiny temporary Git skill library, freezes separate development +and holdout cases, and runs real local subprocesses producing **scripted** answers, +usage and command events. It compares H0, repeated H0 with the same call budget, +and the bounded step. It retains per-trial outputs, grader assertions, traces, +receipts, per-phase totals and a machine-readable summary. Scripted token +differences test accounting and selection; they are not measured model savings. +Wall time measures this local fixture pipeline only. Preparation is reported +separately from execution. No GPU, model API or paid request is involved. +See the [recorded offline example](examples/harness-improvement/README.md), with +the actual aggregate JSON and its explicitly scripted metrics. + +For other reviewed offline fixtures, prepare with: + +```bash +python -m tools.skill_eval_improve prepare --repo \ + --config --output +``` + +The config contains `skill`, `skill_root`, `cases`, `snapshots.H0/INV/DOM`, +`model`, and per-comparison `limits` using the existing comparison formats, +plus: + +| Field | Contract | +| --- | --- | +| `allowlist` | Exact bundle-relative existing `.md` resources permitted to change; no additions/deletions or entrypoint changes | +| `policy.min_success_rate` | Absolute acceptance floor in addition to paired H0 quality | +| `policy.max_regressions` | Maximum allowed paired PASS→FAIL outcomes | +| `policy.quality_weight`, `policy.token_weight` | Finite nonnegative predeclared weights; at least one positive | +| `budget.max_task_calls` | Shared call ceiling across both probes, rerun and holdout | +| `budget.campaign_seconds` | Shared execution deadline; candidate processes use the existing executor/cleanup | +| `budget.token_stop_new_trials` | Shared observed-token threshold; unknown usage also stops admission | + +`execute_offline(output, exact_plan_sha256, client=trusted_fixture)` has no default +client and rejects `CodexClient`. It is a Python interface for trusted offline +test adapters, not a sandbox for arbitrary Python clients. An exclusive start +marker prevents rerunning that output, including after a crash. Recover by +inspecting retained artifacts; never reuse exposed holdout tasks for tuning. +The marker does not police separate campaigns copied to other directories. + +Preparation freezes all possible probe/rerun/holdout comparison plans. Only the +selected branch executes. Report regeneration rechecks frozen inputs, controller +hashes, selection receipts and the original comparison evidence. Modified +evidence cannot preserve a cached acceptance result. Hashes/receipts detect +consistency failures; they are not signatures or a malicious-process sandbox. + +## Costs and evidence boundaries + +Skill Eval summaries now expose input, output, cached-input and total tokens, +command execution counts and wall time. Cached-input tokens are a subset of +input and are not added twice. Missing or malformed usage stays unknown; +missing trial timing cannot become a partial total. `command_calls` counts the +trace's completed command-execution items, not every possible provider tool. +The outer token threshold cannot cap an in-flight request or provider bill. + +The report separates each probe, rerun, holdout, selected-harness execution and +the total observed execution. Proposal authoring cost and monetary cost are +unknown; supplied proposals must not be described as free. The selected +execution figure is a subset of total execution, not an additional charge. +Failed and stopped attempts remain in totals. The independent verifier's FAIL +and runner/evaluation INCONCLUSIVE remain distinct, with retained error causes. + +`development-experience.json` is improver-facing and contains only development +selection/rerun evidence and applicability limitations. It is not executor +skill content and does not write to the knowledge Store. Reviewed Store/CAS +admission and library-on/off transfer experiments are later P2 work. + +Real-model automatic editing requires external filesystem/process confinement, +an explicitly authorized model/budget plan and fresh task validation. Fresh +directories and hashes do not provide that isolation. Efficacy and total-cost +benefits remain **INCONCLUSIVE** until those trials exist. Engine TTFT, TPOT, +GPU cost and service latency still require their separate RunSpec/AB-BA evidence. + +## README diagrams + +The existing animated How It Works diagram describes the engine experiment and +reviewed-knowledge loop, which this change does not alter. Keep that asset and +add an explicitly offline skill-policy path beside it. Architecture now names +the optional policy above Skill Eval and keeps the engine lane separate; the +broader system and module SVGs remain valid at their existing abstraction level. diff --git a/docs/operations.md b/docs/operations.md index 78a8ad6c5..82a20e4fa 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -29,6 +29,7 @@ Effect classes used below: | Adapt existing Flare delivery evidence | [Offline qualification adapter](flare-qualification.md), `python -m inferrsi.model.validators.flare` | Read-only | Existing raw packets and a separate receipt; one HTTP delivery metric, no model/server/device execution | | Evaluate selected skill revisions | [Bounded skill-evaluation driver](skill-evaluation.md) | Experiment-executing | Explicitly authorized real Codex calls, independent task verdicts and private evidence; separate from numeric engine-performance scoring | | Compare skill additions or updates | [Configured skill comparison](skill-eval-comparison.md), `python -m tools.skill_eval_compare` | Read-only preparation / experiment-executing / report-generating | Fixed Git snapshots and one shared exam; explicit plan authorization for execution; JSON + Markdown regeneration makes no model calls | +| Evaluate one bounded offline skill improvement | [Harness improvement](harness-improvement.md), `python -m tools.skill_eval_improve` | Model-free preparation / report-generating; trusted offline execution via Python | One INV/DOM choice, quality/cost gates, mandatory rerun and one-shot holdout; no real-client CLI, automatic edits or promotion | | Plan and report skill PR evaluations | [Skill PR integration](skill-eval-pr.md), `python -m tools.skill_eval_pr` | Read-only planning / authorized experiment-executing / PR-publishing | Trusted mapping, exact operator approval, consumed execution identity and credential-separated publication from retained evidence; no automatic paid trials | | Recover goal progress and the next proposal | [Offline goal checkpoints](goal-checkpoints.md), `python -m tools.goal_checkpoint` | Read-only | Explicit scope/criteria, pinned owner decisions and existing comparison receipts; a disposable view, never execution or acceptance authority | | Summarize campaign gains | `tools/campaign_summary_chart.py`; `report`/`run` figure | Figure-generating | Renders recorded values into one gain figure plus a matching table; validates the specification and computes nothing new | diff --git a/docs/skill-eval-comparison.md b/docs/skill-eval-comparison.md index ccff0d8ee..fe9c934a7 100644 --- a/docs/skill-eval-comparison.md +++ b/docs/skill-eval-comparison.md @@ -8,6 +8,12 @@ generates JSON and Markdown reports. It reuses the existing reader, The [PR integration](skill-eval-pr.md) adds automatic planning, explicit operator authorization and retained-result publication around these interfaces. +The [bounded offline improvement controller](harness-improvement.md) also reuses +these interfaces for INV/DOM probes, successor reruns and held-out acceptance, +with a shared outer execution budget. Summaries include cached-input tokens, +total tokens (input + output, without double-counting cache) and completed +command-execution counts; absent telemetry remains unknown. + ```mermaid flowchart LR A["New: small JSON config"] --> B["New: fixed Git snapshots"] diff --git a/src/inferrsi/harness/improvement.py b/src/inferrsi/harness/improvement.py new file mode 100644 index 000000000..1dba784b5 --- /dev/null +++ b/src/inferrsi/harness/improvement.py @@ -0,0 +1,115 @@ +"""Quality-first selection for one bounded, verifier-guided improvement step. + +The caller supplies receipt-checked Skill Eval reports. This module cannot run +models, read holdout tasks, modify skills or promote knowledge. +""" + +import math +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SelectionPolicy: + min_success_rate: float + max_regressions: int + quality_weight: float + token_weight: float + + def __post_init__(self): + for value in (self.min_success_rate, self.quality_weight, self.token_weight): + if type(value) not in (int, float) or not math.isfinite(value) or value < 0: + raise ValueError("Selection values must be finite and nonnegative") + if self.min_success_rate > 1: + raise ValueError("min_success_rate must be within [0, 1]") + if type(self.max_regressions) is not int or self.max_regressions < 0: + raise ValueError("max_regressions must be a nonnegative integer") + if self.quality_weight + self.token_weight <= 0: + raise ValueError("Selection needs a positive weight") + + +def assess(report: dict, policy: SelectionPolicy, *, split: str, + require_improvement: bool = False) -> dict: + """Reject missing/invalid evidence before comparing quality or cost.""" + reason = None + pairs = report["pairs"] + if report["status"].get("execution_state") != "completed" or not pairs: + reason = "incomplete_execution" + elif any(p[role]["split"] != split for p in pairs for role in ("baseline", "candidate")): + reason = "wrong_split" + elif any(p["outcome"] in ("missing_pair", "inconclusive") for p in pairs): + reason = "inconclusive_evidence" + before, after = (report["conditions"][role] for role in ("baseline", "candidate")) + quality = after["verdicts"].get("PASS", 0) / len(pairs) if pairs else 0 + baseline_quality = before["verdicts"].get("PASS", 0) / len(pairs) if pairs else 0 + tokens = after.get("total_tokens") + baseline_tokens = before.get("total_tokens") + if reason is None: + if tokens is None or baseline_tokens is None: + reason = "unknown_usage" + elif quality < max(policy.min_success_rate, baseline_quality): + reason = "quality_floor" + elif report["counts"]["regression"] > policy.max_regressions: + reason = "regressions" + elif require_improvement and not (quality > baseline_quality or tokens < baseline_tokens): + reason = "no_retained_improvement" + return {"eligible": reason is None, "reason": reason, "success_rate": quality, + "baseline_success_rate": baseline_quality, "total_tokens": tokens, + "baseline_total_tokens": baseline_tokens, + "regressions": report["counts"]["regression"]} + + +def select(reports: dict[str, dict], policy: SelectionPolicy) -> dict: + """Choose INV, DOM, or H0; composition may retain one whole candidate. + + Only strict improvements over the paired H0 are eligible. Equal evidence + retains H0. Pareto filtering precedes normalized weighted squared distance; + a flat dimension contributes zero. Candidate IDs break remaining ties. + """ + if not reports or set(reports) - {"INV", "DOM"}: + raise ValueError("Only one INV and one DOM report are supported") + identities = [{(p["case_id"], p["repeat"]) for p in r["pairs"]} for r in reports.values()] + if any(keys != identities[0] for keys in identities): + raise ValueError("Probes must use matched cases and repeats") + first = next(iter(reports.values())) + if any(r["model"] != first["model"] or r["exam"] != first["exam"] + or r["arms"]["baseline"] != first["arms"]["baseline"] for r in reports.values()): + raise ValueError("Probes must share H0, model/settings and frozen exam") + assessments = {name: assess(report, policy, split="development") + for name, report in reports.items()} + # An incomplete probe makes this bounded decision inconclusive, not a reason + # to cherry-pick a different successful arm. + uncertain = {"incomplete_execution", "wrong_split", "inconclusive_evidence", "unknown_usage"} + if any(a["reason"] in uncertain for a in assessments.values()): + return {"selected": "H0", "reason": "inconclusive_probe", "assessments": assessments, + "pareto": [], "scores": {}} + eligible = {} + for name, a in assessments.items(): + if a["eligible"] and (a["success_rate"] > a["baseline_success_rate"] + or a["total_tokens"] < a["baseline_total_tokens"]): + eligible[name] = a + # Each probe contains the same number of matched H0 trials. Pool those + # independent reference executions instead of cherry-picking a baseline. + eligible["H0"] = { + "success_rate": sum(a["baseline_success_rate"] for a in assessments.values()) + / len(assessments), + "total_tokens": sum(a["baseline_total_tokens"] for a in assessments.values()) + / len(assessments), + } + frontier = {name: a for name, a in eligible.items() if not any( + b["success_rate"] >= a["success_rate"] and b["total_tokens"] <= a["total_tokens"] + and (b["success_rate"] > a["success_rate"] or b["total_tokens"] < a["total_tokens"]) + for b in eligible.values() + )} + scores = {name: 0.0 for name in frontier} + for key, weight, maximize in (("success_rate", policy.quality_weight, True), + ("total_tokens", policy.token_weight, False)): + values = [a[key] for a in frontier.values()] + if values and max(values) > min(values): + low, high = min(values), max(values) + for name, a in frontier.items(): + distance = (high - a[key] if maximize else a[key] - low) / (high - low) + scores[name] += weight * distance ** 2 + selected = min(scores, key=lambda name: (scores[name], name != "H0", name)) + return {"selected": selected, + "reason": "quality_then_cost" if selected != "H0" else "retain_baseline", + "assessments": assessments, "pareto": sorted(frontier), "scores": scores} diff --git a/src/inferrsi/harness/skill_evaluation.py b/src/inferrsi/harness/skill_evaluation.py index eb87c04fe..db18d96f9 100644 --- a/src/inferrsi/harness/skill_evaluation.py +++ b/src/inferrsi/harness/skill_evaluation.py @@ -1,4 +1,5 @@ import json +import math import sys import time from dataclasses import dataclass @@ -89,7 +90,8 @@ def trace_metrics(path: Path) -> dict: item = event["item"] if item["type"] == "command_execution": commands.append(item) - return {"usage": usage, "commands": commands, "errors": errors, "completed_turns": completed} + return {"usage": usage, "commands": commands, "command_count": len(commands), + "errors": errors, "completed_turns": completed} def valid_checks(checks) -> bool: @@ -115,7 +117,6 @@ def summarize(trials: list[dict]) -> dict: groups = {} for condition in sorted({t["condition"] for t in trials}): selected = [t for t in trials if t["condition"] == condition] - usage = [t["observations"]["usage"] for t in selected] groups[condition] = { "trials": len(selected), "verdicts": { @@ -126,10 +127,7 @@ def summarize(trials: list[dict]) -> dict: "elapsed_seconds": sum(t["elapsed_seconds"] for t in selected) if all(t["elapsed_seconds"] is not None for t in selected) else None, - "input_tokens": sum(u["input_tokens"] for u in usage) - if all(u and u.get("input_tokens") is not None for u in usage) else None, - "output_tokens": sum(u["output_tokens"] for u in usage) - if all(u and u.get("output_tokens") is not None for u in usage) else None, + **usage_totals(selected), "cost": None, } paired = [] @@ -152,6 +150,66 @@ def summarize(trials: list[dict]) -> dict: return {"conditions": groups, "pairs": paired, "promotion": "not_evaluated"} +def usage_totals(trials: list[dict]) -> dict: + """Aggregate complete measurements only; cached tokens are part of input.""" + result = {} + usage = [t["observations"].get("usage") for t in trials] + for metric in ("input_tokens", "output_tokens", "cached_input_tokens"): + values = [u.get(metric) if isinstance(u, dict) else None for u in usage] + result[metric] = sum(values) if values and all( + type(value) is int and value >= 0 for value in values + ) else None + result["total_tokens"] = ( + result["input_tokens"] + result["output_tokens"] + if result["input_tokens"] is not None and result["output_tokens"] is not None else None + ) + commands = [t["observations"].get("command_count") for t in trials] + result["command_calls"] = sum(commands) if commands and all( + type(value) is int and value >= 0 for value in commands + ) else None + return result + + +@dataclass +class TrialBudget: + """Shared outer-loop call/deadline limits and observed-token admission stop. + + Tokens stop new trials, not an in-flight provider request. Unknown usage + stops admission too. The caller owns this object, never the candidate. + """ + + max_calls: int + deadline: float + token_stop: int + calls: int = 0 + tokens: int = 0 + usage_complete: bool = True + + def __post_init__(self): + if any(type(n) is not int or n <= 0 for n in (self.max_calls, self.token_stop)): + raise ValueError("Call and token limits must be positive integers") + if not math.isfinite(self.deadline): + raise ValueError("Deadline must be finite") + + def stop_reason(self) -> str | None: + if not self.usage_complete: + return "unknown_usage" + if time.monotonic() >= self.deadline: + return "time_budget" + if self.calls >= self.max_calls: + return "call_budget" + if self.tokens >= self.token_stop: + return "token_budget" + return None + + def record(self, trial: dict) -> None: + self.calls += 1 + tokens = usage_totals([trial])["total_tokens"] + self.usage_complete &= tokens is not None + if tokens is not None: + self.tokens += tokens + + def export_bundle(reader: KnowledgeReader, reference: str, destination: Path) -> dict: bundle = reader.read(reference) for name, data in bundle.content.items(): diff --git a/tests/fixtures/skill_eval_improvement_client.py b/tests/fixtures/skill_eval_improvement_client.py new file mode 100644 index 000000000..3559ba4ac --- /dev/null +++ b/tests/fixtures/skill_eval_improvement_client.py @@ -0,0 +1,19 @@ +"""Scripted subprocess telemetry; no model, network, or billed tokens.""" + +import json +import sys +from pathlib import Path + +mode, final, answer, tokens = sys.argv[1:] +output = json.loads(Path(answer).read_text(encoding="utf-8")) +if mode == "incorrect": + output["claim_scope"] = "production" +Path(final).write_text(json.dumps(output), encoding="utf-8") +for _ in range(2): + print(json.dumps({"type": "item.completed", "item": { + "type": "command_execution", "command": "fixture-read", "exit_code": 0}})) +usage = {"input_tokens": int(tokens) - 10, "output_tokens": 10, "cached_input_tokens": 20} +if mode == "malformed_usage": + usage = [1] +print(json.dumps({"type": "turn.completed", "usage": None if mode == "unknown" else usage})) +raise SystemExit(7 if mode == "runner_error" else 0) diff --git a/tests/fixtures/skill_eval_improvement_demo.py b/tests/fixtures/skill_eval_improvement_demo.py new file mode 100644 index 000000000..0154188c5 --- /dev/null +++ b/tests/fixtures/skill_eval_improvement_demo.py @@ -0,0 +1,75 @@ +"""Reproduce H0 / equal-call-budget H0 / bounded-loop offline controls.""" + +import argparse +import json +import time +from pathlib import Path + +from inferrsi.infra.artifacts import sha256, write_json +from tests.fixtures.skill_eval_improvement_support import ImprovementClient, configuration +from tools import skill_eval_compare as comparison +from tools import skill_eval_improve as improvement + + +def run(output: Path) -> dict: + output.mkdir(parents=True, exist_ok=False) + rows = [] + for name, repeats in (("H0", 3), ("repeated-H0", 12), ("bounded", 3)): + root = output / name + started = time.monotonic() + config_path, repo = configuration(root, repeats=repeats, max_calls=48) + client = ImprovementClient(root) + directory = root / "run" + if name == "bounded": + digest = improvement.prepare(config_path, repo, directory) + else: + config = comparison.read_json(config_path) + tasks = comparison.read_json(root / "cases.json") + write_json(root / "cases.json", [t for t in tasks if t["split"] == "holdout"]) + control = {k: config[k] for k in ("skill", "skill_root", "cases", "model", "limits")} + control.update(mode="update", snapshots={ + "baseline": config["snapshots"]["H0"], "candidate": config["snapshots"]["H0"]}) + write_json(config_path, control) + digest = comparison.prepare(config_path, repo, directory) + preparation = time.monotonic() - started + started = time.monotonic() + if name == "bounded": + result = improvement.execute_offline(directory, digest, client=client) + totals = result["observed_execution"] + heldout = comparison.read_report(directory / "comparisons/holdout-INV") + else: + result = comparison.execute(directory, digest, client=client) + heldout = result + conditions = result["conditions"].values() + totals = {"calls": len(client.calls), + "total_tokens": sum(c["total_tokens"] for c in conditions), + "command_calls": sum(c["command_calls"] for c in conditions)} + execution = time.monotonic() - started + rows.append({ + "condition": name, "preparation_seconds": preparation, + "execution_seconds": execution, "calls": len(client.calls), + "scripted_total_tokens": totals["total_tokens"], + "scripted_command_calls": totals["command_calls"], + "holdout_candidate_trials": heldout["conditions"]["candidate"]["trials"], + "holdout_candidate_verdicts": heldout["conditions"]["candidate"]["verdicts"], + "holdout_regressions": heldout["counts"]["regression"], + "effective_candidate": result.get("effective_candidate", "H0"), + "plan_sha256": digest, "report_sha256": sha256(directory / "report.json"), + }) + summary = { + "kind": "offline_fixture_control", "rows": rows, + "scope": "Actual subprocess calls and pipeline wall time; scripted answers and tokens. " + "H0 is a paired H0/H0 control, repeated-H0 has the bounded loop's 48-call budget. " + "No best-of-N answer selection. Different repeat counts are not efficacy estimates.", + "model_calls": 0, "monetary_cost": None, "proposal_authoring_cost": None, + "efficacy_verdict": "INCONCLUSIVE", "engine_performance_verdict": "INCONCLUSIVE", + } + write_json(output / "summary.json", summary) + return summary + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(run(args.output.resolve()), indent=2)) diff --git a/tests/fixtures/skill_eval_improvement_support.py b/tests/fixtures/skill_eval_improvement_support.py new file mode 100644 index 000000000..ae8e515b8 --- /dev/null +++ b/tests/fixtures/skill_eval_improvement_support.py @@ -0,0 +1,79 @@ +import copy +import json +import sys + +from inferrsi.infra.artifacts import write_json +from tests.fixtures.skill_eval_compare_support import DATA, ROOT, git + + +def configuration(root, *, repeats=1, max_calls=32, token_stop=100000): + root.mkdir(parents=True, exist_ok=True) + repo = root / "repo" + repo.mkdir() + git(repo, "init", "--quiet") + folder = repo / "library/audit" + folder.mkdir(parents=True) + body = "---\nname: audit\ndescription: offline test fixture\n---\nH0\n" + (folder / "SKILL.md").write_text(body, encoding="utf-8") + (folder / "notes.md").write_text("H0 notes\n", encoding="utf-8") + git(repo, "add", ".") + git(repo, "commit", "--quiet", "-m", "H0 fixture") + baseline = git(repo, "rev-parse", "HEAD") + (folder / "SKILL.md").write_text(body + "INV\n", encoding="utf-8") + git(repo, "add", ".") + git(repo, "commit", "--quiet", "-m", "INV fixture") + inv = git(repo, "rev-parse", "HEAD") + (folder / "SKILL.md").write_text(body, encoding="utf-8") + (folder / "notes.md").write_text("DOM notes\n", encoding="utf-8") + git(repo, "add", ".") + git(repo, "commit", "--quiet", "-m", "DOM fixture") + dom = git(repo, "rev-parse", "HEAD") + tasks = [{**copy.deepcopy(DATA["task"]), "id": f"{split}-{i}", "split": split} + for split in ("development", "holdout") for i in (1, 2)] + write_json(root / "cases.json", tasks) + config = { + "skill": "audit", "skill_root": "library", "cases": "cases.json", + "snapshots": {name: {"git_revision": rev} + for name, rev in (("H0", baseline), ("INV", inv), ("DOM", dom))}, + "model": {"name": "offline-fixture", "effort": "fixed"}, + "limits": {"repeats": repeats, "max_task_calls": 8 * repeats, + "timeout_seconds": 20, "campaign_seconds": 120, + "token_stop_new_trials": token_stop}, + "budget": {"max_task_calls": max_calls, "campaign_seconds": 120, + "token_stop_new_trials": token_stop}, + "policy": {"min_success_rate": 1.0, "max_regressions": 0, + "quality_weight": 1.0, "token_weight": 1.0}, + "allowlist": ["audit/SKILL.md", "audit/notes.md"], + } + write_json(root / "config.json", config) + return root / "config.json", repo + + +class ImprovementClient: + """A trusted fixture adapter; scripted outcomes must never be called efficacy.""" + + timeout_seconds = 20 + + def __init__(self, root, overrides=None, token_overrides=None): + self.answer = root / "answer.json" + write_json(self.answer, DATA["cases"][0]["answer"]) + self.overrides = overrides or {} + self.token_overrides = token_overrides or {} + self.calls = [] + + def command(self, workspace, final, schema, prompt): + task = json.loads((workspace / "task.json").read_text(encoding="utf-8")) + body = (workspace / "knowledge/audit/SKILL.md").read_text(encoding="utf-8") + notes = (workspace / "knowledge/audit/notes.md").read_text(encoding="utf-8") + candidate = "INV" if "INV" in body else "DOM" if "DOM" in notes else "H0" + stage = final.parents[2].name + mode = "incorrect" if candidate == "DOM" and task["id"].endswith("-1") else "correct" + mode = self.overrides.get((stage, candidate), mode) + tokens = self.token_overrides.get((stage, candidate), + {"H0": 120, "INV": 60, "DOM": 40}[candidate]) + self.calls.append({"stage": stage, "candidate": candidate, "case": task["id"], + "files": sorted(p.relative_to(workspace).as_posix() + for p in workspace.rglob("*") if p.is_file()), + "prompt": prompt}) + return (sys.executable, str(ROOT / "tests/fixtures/skill_eval_improvement_client.py"), + mode, str(final), str(self.answer), str(tokens)) diff --git a/tests/test_skill_improvement.py b/tests/test_skill_improvement.py new file mode 100644 index 000000000..8c83ea5c2 --- /dev/null +++ b/tests/test_skill_improvement.py @@ -0,0 +1,209 @@ +import json +import subprocess + +import pytest + +from inferrsi.harness.improvement import SelectionPolicy, select +from inferrsi.harness.skill_evaluation import CodexClient, TrialBudget, usage_totals +from tests.fixtures.skill_eval_improvement_support import ImprovementClient, configuration +from tools import skill_eval_improve as improve +from tools.skill_eval_compare import require_git_version + + +@pytest.fixture +def prepared(tmp_path): + try: + require_git_version(subprocess.check_output(("git", "--version"), text=True)) + except ValueError as error: + pytest.skip(str(error)) + config, repo = configuration(tmp_path) + output = tmp_path / "run" + digest = improve.prepare(config, repo, output) + return output, digest, ImprovementClient(tmp_path) + + +def test_selects_cost_saving_only_after_quality_and_rerun(prepared): + output, digest, client = prepared + result = improve.execute_offline(output, digest, client=client) + assert result["accepted"] and result["effective_candidate"] == "INV" + assert result["selection"]["assessments"]["DOM"]["reason"] == "quality_floor" + assert result["observed_execution"]["calls"] == 16 + assert result["observed_execution"]["total_tokens"] == 1400 + assert result["observed_execution"]["cached_input_tokens"] == 320 + assert result["observed_execution"]["command_calls"] == 32 + assert result["selected_execution"]["total_tokens"] == 120 + assert result["observed_execution"]["cost"] is None + assert result["proposal_cost"] is None + stages = [c["stage"] for c in client.calls] + assert stages == sum(([stage] * 4 for stage in ( + "probe-INV", "probe-DOM", "rerun-INV", "holdout-INV")), []) + for call in client.calls: + assert call["files"] == ["knowledge/audit/SKILL.md", "knowledge/audit/notes.md", + "task.json"] + if call["stage"].startswith("probe"): + assert call["case"].startswith("development") + assert "holdout" not in call["prompt"] + experience = json.loads((output / "development-experience.json").read_text()) + assert "holdout" not in json.dumps(experience) + with pytest.raises(FileExistsError): + improve.execute_offline(output, digest, client=client) + assert len(client.calls) == 16 + + +@pytest.mark.parametrize("stage", ["rerun-INV", "holdout-INV"]) +@pytest.mark.parametrize("mode", ["incorrect", "unknown", "runner_error"]) +def test_bad_successor_or_holdout_retains_h0(prepared, stage, mode): + output, digest, client = prepared + client.overrides[stage, "INV"] = mode + result = improve.execute_offline(output, digest, client=client) + assert not result["accepted"] and result["effective_candidate"] == "H0" + if stage == "rerun-INV": + assert not any(c["stage"].startswith("holdout") for c in client.calls) + + +@pytest.mark.parametrize("stage", ["rerun-INV", "holdout-INV"]) +def test_cost_only_improvement_must_survive_rerun_and_holdout(prepared, stage): + output, digest, client = prepared + client.token_overrides[stage, "INV"] = 240 + result = improve.execute_offline(output, digest, client=client) + assert not result["accepted"] and result["effective_candidate"] == "H0" + + +@pytest.mark.parametrize("mode", ["unknown", "malformed_usage"]) +def test_unknown_probe_usage_stops_entire_step_and_keeps_h0(prepared, mode): + output, digest, client = prepared + client.overrides["probe-INV", "H0"] = mode + result = improve.execute_offline(output, digest, client=client) + assert not result["accepted"] and result["selection"]["selected"] == "H0" + assert len(client.calls) == 1 and result["budget"]["stop_reason"] == "unknown_usage" + + +def test_tampered_probe_evidence_invalidates_cached_acceptance(prepared): + output, digest, client = prepared + assert improve.execute_offline(output, digest, client=client)["accepted"] + trace = next((output / "comparisons/probe-INV/results").glob("*/agent/stdout.log")) + trace.write_text("tampered", encoding="utf-8") + replay = improve.report(output) + assert not replay["accepted"] and not replay["retained_selection_consistent"] + + +def test_frozen_selection_and_hidden_input_tampering_are_rejected(prepared): + output, digest, client = prepared + hidden = output / "inputs/holdout.json" + hidden.write_text("[]", encoding="utf-8") + with pytest.raises(ValueError, match="Frozen improvement input changed"): + improve.execute_offline(output, digest, client=client) + assert not client.calls + + +def test_real_client_cannot_use_offline_entry(prepared): + output, digest, _ = prepared + with pytest.raises(ValueError, match="Real-model"): + improve.execute_offline(output, digest, client=CodexClient("unused", "unused", 1)) + + +@pytest.mark.parametrize("limit", [1, 3, 9]) +def test_outer_budget_bounds_all_phases(tmp_path, limit): + try: + require_git_version(subprocess.check_output(("git", "--version"), text=True)) + except ValueError as error: + pytest.skip(str(error)) + config, repo = configuration(tmp_path, max_calls=limit) + output = tmp_path / "run" + digest = improve.prepare(config, repo, output) + client = ImprovementClient(tmp_path) + result = improve.execute_offline(output, digest, client=client) + assert len(client.calls) == limit and not result["accepted"] + assert result["budget"]["stop_reason"] == "call_budget" + + +def test_token_threshold_is_shared_across_probes_and_rerun(tmp_path): + try: + require_git_version(subprocess.check_output(("git", "--version"), text=True)) + except ValueError as error: + pytest.skip(str(error)) + config, repo = configuration(tmp_path, token_stop=700) + output = tmp_path / "run" + digest = improve.prepare(config, repo, output) + client = ImprovementClient(tmp_path) + result = improve.execute_offline(output, digest, client=client) + # Two probes consume 680 scripted tokens. One rerun request crosses the + # admission threshold, then no new request (including holdout) may start. + assert len(client.calls) == 9 and client.calls[-1]["stage"] == "rerun-INV" + assert result["budget"]["stop_reason"] == "token_budget" + assert result["effective_candidate"] == "H0" + + +def test_candidate_cannot_edit_an_unlisted_resource(tmp_path): + try: + require_git_version(subprocess.check_output(("git", "--version"), text=True)) + except ValueError as error: + pytest.skip(str(error)) + config, repo = configuration(tmp_path) + data = json.loads(config.read_text()) + data["allowlist"] = ["audit/SKILL.md"] + config.write_text(json.dumps(data)) + with pytest.raises(ValueError, match="outside the Markdown allowlist"): + improve.prepare(config, repo, tmp_path / "run") + + +def test_usage_missing_and_cached_subset_are_not_double_counted(): + trial = {"observations": {"usage": {"input_tokens": 10, "output_tokens": 2, + "cached_input_tokens": 8}, "command_count": 3}} + assert usage_totals([trial])["total_tokens"] == 12 + unknown = {"observations": {"usage": {"input_tokens": 10}}} + assert usage_totals([trial, unknown])["total_tokens"] is None + assert usage_totals([trial, unknown])["cached_input_tokens"] is None + assert usage_totals([trial, unknown])["command_calls"] is None + + +@pytest.mark.parametrize("changes", [{"min_success_rate": float("nan")}, + {"min_success_rate": 2}, {"max_regressions": True}, + {"token_weight": -1}, + {"quality_weight": 0, "token_weight": 0}]) +def test_invalid_policy_rejected(changes): + values = dict(min_success_rate=1, max_regressions=0, quality_weight=1, token_weight=1) + with pytest.raises(ValueError): + SelectionPolicy(**(values | changes)) + + +def test_deadline_and_unknown_usage_are_external_admission_stops(): + budget = TrialBudget(1, 0, 1) + assert budget.stop_reason() == "time_budget" + budget.record({"observations": {"usage": None}}) + assert budget.stop_reason() == "unknown_usage" + + +def policy_report(baseline_passes, candidate_passes, tokens, regressions=0): + return { + "pairs": [{"case_id": str(i), "repeat": 1, "outcome": "unchanged_correct", + "baseline": {"split": "development"}, "candidate": {"split": "development"}} + for i in range(4)], + "status": {"execution_state": "completed"}, "model": {"name": "fixture"}, + "exam": {"cases": "fixed"}, "arms": {"baseline": {"bundle": "fixed"}}, + "conditions": {"baseline": {"verdicts": {"PASS": baseline_passes}, "total_tokens": 100}, + "candidate": {"verdicts": {"PASS": candidate_passes}, + "total_tokens": tokens}}, + "counts": {"regression": regressions}, + } + + +def test_cost_weight_can_retain_h0_over_higher_quality_expensive_candidates(): + reports = {"INV": policy_report(2, 4, 1000), "DOM": policy_report(2, 3, 800)} + assert select(reports, SelectionPolicy(0.5, 0, 0, 1))["selected"] == "H0" + assert select(reports, SelectionPolicy(0.5, 0, 1, 0))["selected"] == "INV" + + +def test_same_quality_regression_fails_before_cost_selection(): + report = policy_report(2, 2, 50, regressions=1) + decision = select({"INV": report}, SelectionPolicy(0.5, 0, 1, 1)) + assert decision["selected"] == "H0" + assert decision["assessments"]["INV"]["reason"] == "regressions" + + +def test_equal_candidates_keep_h0_and_mismatched_probes_are_rejected(): + reports = {"INV": policy_report(4, 4, 100), "DOM": policy_report(4, 4, 100)} + assert select(reports, SelectionPolicy(1, 0, 1, 1))["selected"] == "H0" + reports["DOM"]["exam"] = {"cases": "different"} + with pytest.raises(ValueError, match="share H0"): + select(reports, SelectionPolicy(1, 0, 1, 1)) diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index 1f924d034..12a0cad5f 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -9,7 +9,14 @@ import time from pathlib import Path -from inferrsi.harness.skill_evaluation import CodexClient, export_bundle, run_trial, summarize +from inferrsi.harness.skill_evaluation import ( + CodexClient, + TrialBudget, + export_bundle, + run_trial, + summarize, + usage_totals, +) from inferrsi.infra.artifacts import atomic_write_json, sha256, write_json from inferrsi.knowledge_operations import KnowledgeReader, safe_path from tools.skill_eval_pilot import PROMPT, SCHEMA, observed_tokens @@ -384,7 +391,8 @@ def receipt_error(raw_record: bytes, receipt_path: Path, label: str) -> str | No return None -def execute(output: Path, authorization: str, *, client=None) -> dict: +def execute(output: Path, authorization: str, *, client=None, + budget: TrialBudget | None = None) -> dict: if authorization != sha256(output / "plan/freeze.json"): raise ValueError("Execution requires authorization for this exact plan SHA-256") plan = load_plan(output) @@ -398,8 +406,12 @@ def execute(output: Path, authorization: str, *, client=None) -> dict: {"started_at": time.time(), "client_runtime": runtime}) tasks = {task["id"]: task for task in read_json(output / "plan/tasks.json")} deadline = time.monotonic() + plan["limits"]["campaign_seconds"] + if budget is not None: + deadline = min(deadline, budget.deadline) tokens, completed, stop_reason = 0, 0, None for entry in plan["schedule"]: + if budget is not None and (stop_reason := budget.stop_reason()): + break if time.monotonic() >= deadline or tokens >= plan["limits"]["token_stop_new_trials"]: stop_reason = "time_budget" if time.monotonic() >= deadline else "token_budget" break @@ -425,6 +437,8 @@ def execute(output: Path, authorization: str, *, client=None) -> dict: atomic_write_json(directory / "receipt.json", {"result_sha256": sha256(directory / "result.json")}) tokens += observed_tokens(result["observations"]["usage"]) + if budget is not None: + budget.record(result) completed += 1 if result["execution_state"] != "completed": stop_reason = "client_" + result["execution_state"] @@ -473,7 +487,10 @@ def retained_trial(output: Path, entry: dict) -> dict: # Select report fields; raw commands and private workspace paths stay in local artifacts. selected = {key: record[key] for key in ("task_id", "condition", "repeat", "split", "verdict", "checks", "elapsed_seconds", "observations", "inputs_unchanged")} - selected["observations"] = {"usage": record["observations"]["usage"]} + selected["observations"] = { + "usage": record["observations"]["usage"], + "command_count": record["observations"].get("command_count"), + } error = record.get("evaluation_error") if data is None: error = error or "missing_retained_output" @@ -512,11 +529,9 @@ def condition_summary(trials: list[dict]) -> dict: "elapsed_seconds": None, "cost": None}) group["expected_trials"] = len(selected) group["missing_trials"] = sum(t["missing"] for t in selected) - for metric in ("input_tokens", "output_tokens"): - usage = [t["observations"]["usage"] for t in selected] - group[metric] = sum(u[metric] for u in usage) if all( - u is not None and u.get(metric) is not None for u in usage - ) and usage else None + group.update(usage_totals(selected)) + if any(t["missing"] or t["elapsed_seconds"] is None for t in selected): + group["elapsed_seconds"] = None return groups diff --git a/tools/skill_eval_improve.py b/tools/skill_eval_improve.py new file mode 100644 index 000000000..3f02eeef9 --- /dev/null +++ b/tools/skill_eval_improve.py @@ -0,0 +1,241 @@ +"""Offline first slice of issue #93, above the existing comparison runner. + +Preparation and reporting are model-free. Execution is a Python interface for a +trusted fixed-output client; there is deliberately no real-client CLI here. +""" + +import argparse +import json +import time +from pathlib import Path + +from inferrsi.harness.improvement import SelectionPolicy, assess, select +from inferrsi.harness.skill_evaluation import CodexClient, TrialBudget, usage_totals +from inferrsi.infra.artifacts import sha256, write_json +from inferrsi.knowledge_operations import safe_path +from tools import skill_eval_compare as comparison + +ROOT = Path(__file__).resolve().parents[1] +CONTROLLERS = (*comparison.CONTROLLERS, "tools/skill_eval_improve.py", + "src/inferrsi/harness/improvement.py", + "src/inferrsi/knowledge_operations.py", "tools/skill_eval_verifier.py") + + +def prepare(config_path: Path, repo: Path, output: Path) -> str: + config = comparison.read_json(config_path) + required = {"skill", "skill_root", "cases", "snapshots", "model", "limits", + "policy", "allowlist", "budget"} + if set(config) != required or set(config["snapshots"]) != {"H0", "INV", "DOM"}: + raise ValueError("Declare H0, one INV, one DOM and the supported configuration fields") + SelectionPolicy(**config["policy"]) + budget = config["budget"] + if set(budget) != {"max_task_calls", "campaign_seconds", "token_stop_new_trials"} or any( + type(n) is not int or n <= 0 for n in budget.values() + ): + raise ValueError("Declare positive outer call, time and observed-token limits") + allowlist = config["allowlist"] + if not isinstance(allowlist, list) or not allowlist or len(set(allowlist)) != len(allowlist): + raise ValueError("Declare a nonempty unique resource allowlist") + for path in allowlist: + if not safe_path(path).endswith(".md"): + raise ValueError("Only explicitly allowlisted Markdown resources may change") + tasks = comparison.read_json(config_path.parent / config["cases"]) + if any(t["split"] not in ("development", "holdout") for t in tasks): + raise ValueError("Improvement requires development/holdout splits only") + splits = {split: [t for t in tasks if t["split"] == split] + for split in ("development", "holdout")} + if not all(splits.values()) or len({t["id"] for t in tasks}) != len(tasks): + raise ValueError("Nonempty disjoint development and holdout cases are required") + output.mkdir(parents=True, exist_ok=False) + (output / "inputs").mkdir() + (output / "results").mkdir() + for split, cases in splits.items(): + write_json(output / "inputs" / (split + ".json"), cases) + stages = [("probe-INV", "INV", "development"), ("probe-DOM", "DOM", "development")] + stages += [("rerun-" + name, name, "development") for name in ("H0", "INV", "DOM")] + stages += [("holdout-" + name, name, "holdout") for name in ("INV", "DOM")] + plans = {} + for stage, name, split in stages: + child_config = {key: config[key] for key in ("skill", "skill_root", "model", "limits")} + child_config.update(mode="update", cases=split + ".json", snapshots={ + "baseline": config["snapshots"]["H0"], "candidate": config["snapshots"][name]}) + path = output / "inputs" / (stage + ".json") + write_json(path, child_config) + directory = output / "comparisons" / stage + plans[stage] = comparison.prepare(path, repo, directory) + plan = comparison.load_plan(directory) + if plan["arms"]["baseline"]["bundle"]["entrypoint"] != ( + plan["arms"]["candidate"]["bundle"]["entrypoint"] + ): + raise ValueError("Candidate may not change the skill entrypoint") + if any(c["path"] not in allowlist for c in plan["bundle_changes"]): + raise ValueError("Candidate changed a resource outside the Markdown allowlist") + if any(c["baseline"] is None or c["candidate"] is None for c in plan["bundle_changes"]): + raise ValueError("The first slice supports edits to existing resources only") + plan = {"format_version": 1, "config": config, "plans": plans, + "inputs": {p.name: sha256(p) for p in (output / "inputs").iterdir()}, + "controllers": {name: sha256(ROOT / name) for name in CONTROLLERS}, + "execution": "trusted_offline_client_only", "composition": "select_INV_DOM_or_H0", + "promotion": "not_evaluated"} + write_json(output / "freeze.json", plan) + return sha256(output / "freeze.json") + + +def load_plan(output: Path) -> dict: + plan = comparison.read_json(output / "freeze.json") + for name, digest in plan["controllers"].items(): + if sha256(ROOT / name) != digest: + raise ValueError("Improvement controller changed after freeze: " + name) + if {p.name for p in (output / "inputs").iterdir()} != set(plan["inputs"]): + raise ValueError("Frozen input set changed") + for name, digest in plan["inputs"].items(): + if sha256(output / "inputs" / name) != digest: + raise ValueError("Frozen improvement input changed: " + name) + for stage, digest in plan["plans"].items(): + directory = output / "comparisons" / stage + if sha256(directory / "plan/freeze.json") != digest: + raise ValueError("Frozen comparison plan changed: " + stage) + comparison.load_plan(directory) + return plan + + +def save_record(path: Path, data: dict) -> None: + write_json(path, data) + write_json(path.with_suffix(".receipt.json"), {"result_sha256": sha256(path)}) + + +def read_record(path: Path) -> dict: + error = comparison.receipt_error(path.read_bytes(), path.with_suffix(".receipt.json"), + "improvement") + if error: + raise ValueError(error) + return comparison.read_json(path) + + +def execute_offline(output: Path, authorization: str, *, client) -> dict: + """Run only a caller-reviewed offline adapter, never infer a model client. + + This is not a sandbox for arbitrary adapters. Real-model generation and + execution require external read/write isolation and separate qualification. + """ + if isinstance(client, CodexClient): + raise ValueError("Real-model execution is outside the offline improvement slice") + if authorization != sha256(output / "freeze.json"): + raise ValueError("Execution requires the exact improvement plan SHA-256") + plan = load_plan(output) + # Exclusive marker consumes the whole step, even after a crash. No retry may + # tune a new candidate against this output's already exposed holdout. + with (output / "results/started.json").open("x", encoding="utf-8") as stream: + json.dump({"plan_sha256": authorization}, stream) + started = time.monotonic() + limits = plan["config"]["budget"] + budget = TrialBudget(limits["max_task_calls"], started + limits["campaign_seconds"], + limits["token_stop_new_trials"]) + policy = SelectionPolicy(**plan["config"]["policy"]) + stages = [] + + def run(stage): + stages.append(stage) + return comparison.execute(output / "comparisons" / stage, plan["plans"][stage], + client=client, budget=budget) + + probes = {name: run("probe-" + name) for name in ("INV", "DOM")} + decision = select(probes, policy) + selected = decision["selected"] + # Freeze before successor execution, then again verify before the holdout. + decision["plan_sha256"] = authorization + save_record(output / "results/selection.json", decision) + rerun = run("rerun-" + selected) + rerun_assessment = assess(rerun, policy, split="development", + require_improvement=selected != "H0") + if selected != "H0" and rerun_assessment["eligible"]: + load_plan(output) + if read_record(output / "results/selection.json") != decision: + raise ValueError("Selection changed before holdout") + run("holdout-" + selected) + save_record(output / "results/state.json", { + "plan_sha256": authorization, "stages": stages, + "selection_sha256": sha256(output / "results/selection.json"), + "elapsed_seconds": time.monotonic() - started, + "budget": {"calls": budget.calls, "observed_tokens": budget.tokens, + "usage_complete": budget.usage_complete, "stop_reason": budget.stop_reason()}, + }) + return report(output) + + +def report(output: Path) -> dict: + """Recheck retained receipts and evidence; do not trust a cached verdict.""" + plan = load_plan(output) + state = read_record(output / "results/state.json") + if state["plan_sha256"] != sha256(output / "freeze.json"): + raise ValueError("Completed run belongs to a different improvement plan") + if state["selection_sha256"] != sha256(output / "results/selection.json"): + raise ValueError("Frozen selection changed") + decision = read_record(output / "results/selection.json") + reports = {stage: comparison.read_report(output / "comparisons" / stage) + for stage in state["stages"]} + policy = SelectionPolicy(**plan["config"]["policy"]) + replay = select({name: reports["probe-" + name] for name in ("INV", "DOM")}, policy) + selected = decision["selected"] + consistent = all(decision[key] == value for key, value in replay.items()) + rerun = assess(reports["rerun-" + selected], policy, split="development", + require_improvement=selected != "H0") + holdout = (assess(reports["holdout-" + selected], policy, split="holdout", + require_improvement=True) + if "holdout-" + selected in reports else None) + accepted = consistent and selected != "H0" and rerun["eligible"] and ( + holdout is not None and holdout["eligible"]) + phase_usage = {} + all_trials = [] + for stage, result in reports.items(): + trials = [p[role] for p in result["pairs"] for role in ("baseline", "candidate") + if not p[role]["missing"]] + all_trials.extend(trials) + phase_usage[stage] = {"calls": len(trials), **usage_totals(trials), + "elapsed_seconds": sum(t["elapsed_seconds"] for t in trials) + if all(t["elapsed_seconds"] is not None for t in trials) else None} + result = { + "plan_sha256": state["plan_sha256"], "selection": decision, + "retained_selection_consistent": consistent, "rerun": rerun, "holdout": holdout, + "accepted": accepted, "effective_candidate": selected if accepted else "H0", + "phase_usage": phase_usage, "observed_execution": { + "calls": len(all_trials), **usage_totals(all_trials), + "elapsed_seconds": state["elapsed_seconds"], "cost": None}, + "budget": state["budget"], + "proposal_cost": None, + "proposal_cost_note": "Caller-prepared snapshots; authoring cost is unknown and excluded.", + "selected_execution": reports["rerun-" + selected]["conditions"]["candidate"], + "promotion": "not_evaluated", + "claim": "Offline control-flow evidence only. Fixture tokens are scripted, not billed. " + "No measured model efficacy, end-to-end savings, sandbox isolation or engine speedup.", + } + write_json(output / "report.json", result) + # Improver-facing experience deliberately excludes holdout scores and answers. + write_json(output / "development-experience.json", { + "kind": "improvement_experience", "plan_sha256": state["plan_sha256"], + "selection": replay, "rerun": rerun, "applicability": "offline performance-report audit", + "limitations": ["Candidate only; no automatic Store write or promotion", + "No cross-task transfer evaluation", "Caller-authored proposals"], + }) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("prepare", "report")) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--config", type=Path) + parser.add_argument("--repo", type=Path) + args = parser.parse_args() + if args.mode == "prepare": + if args.config is None or args.repo is None: + parser.error("prepare requires --config and --repo") + result = {"plan_sha256": prepare(args.config.resolve(), args.repo.resolve(), + args.output.resolve())} + else: + result = report(args.output.resolve()) + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/skill_eval_pilot.py b/tools/skill_eval_pilot.py index d52dc7e2f..eb78b1154 100644 --- a/tools/skill_eval_pilot.py +++ b/tools/skill_eval_pilot.py @@ -282,11 +282,11 @@ def recover_attempts(output: Path, schedule: list[dict]) -> list[dict]: def observed_tokens(usage: dict | None) -> int: # Count observed token fields only; report completeness separately in status and summary. - if usage is None: + if not isinstance(usage, dict): return 0 return sum( value for key, value in usage.items() - if key in ("input_tokens", "output_tokens") and value is not None + if key in ("input_tokens", "output_tokens") and type(value) is int and value >= 0 )