diff --git a/commit0/SAMPLING_RESULTS.md b/commit0/SAMPLING_RESULTS.md new file mode 100644 index 0000000..0e6f1cb --- /dev/null +++ b/commit0/SAMPLING_RESULTS.md @@ -0,0 +1,62 @@ +# Sampling results — agent value-add significance (Tier-1, 2026-06) + +Step 4 of the validation plan: *characterize agent value-add variance and test significance, +so "value-add" claims are distinguishable from LLM run-to-run noise.* Method: K=5 reps/cell +via `repeat_runner` (valid-rep gating), each rep scored canonically by `score_branch` +(commit0 full-suite), analyzed by `stats_analyze` (bootstrap 95% CIs, paired value-add Δ vs +the single-shot baseline, BH-FDR q=0.05, sign-stability). **OpenAI (gpt-5.4), default +temperature** (so variance matches the published numbers). Cost: **~$17.5**. + +## cachetools (near-saturated lib) +| arch | pass% [95% CI] | Δ vs single_shot [CI] | sig | +|---|---|---|---| +| aider | 100.0 [100,100] | +13.0 [+2,+25] | | +| smolagents | 99.7 [99,100] | +12.7 [+1,+24] | | +| kaizen_delta | 95.5 [93,97] | +8.6 [−3,+21] | | +| reflexion | 82.9 [81,85] | −4.1 [−16,+8] | | +| single_shot (baseline) | 87.0 [75,98] | — | | + +**0/4 significant after BH-FDR.** Complementary pattern is visible (aider/smolagents win, +reflexion regresses), but effects are small and the **baseline is noisy (87% ±12pp)**, so +nothing survives multiple-comparison correction at K=5. Agentic archs are far more +*reproducible* than single-shot (σ≈0 vs ≈12pp) — a finding in itself. + +## voluptuous (floor lib — architectural unlock) +| arch | pass% [95% CI] | Δ vs single_shot [CI] | sig | +|---|---|---|---| +| **aider** | **88.7 [87,91]** | **+82.6 [+70,+90]** | **\*** | +| smolagents | 40.0 [0,80] | +33.8 [−6,+80] | bimodal, n.s. | +| reflexion | 24.7 [0,59] | +18.5 [−11,+51] | bimodal, n.s. | +| kaizen_delta | 0.0 [0,0] | −6.2 [−19,+0] | no unlock | +| single_shot (baseline) | 6.2 [0,19] | — | | + +**1/4 significant after BH-FDR (aider).** + +## What the sampling establishes (and corrects) +1. **A real, large, reliable value-add:** aider unlocks voluptuous at **+82.6pp, significant + even at K=5** — the headline "agent value-add" claim, now with a CI. +2. **Bimodal pseudo-unlocks exposed:** smolagents/reflexion *sometimes* crack voluptuous and + *sometimes* crash (149/149 vs 0/2 across reps) → wide CIs → **not significant**. A single + run would have over-claimed these as reliable unlocks; sampling shows they aren't. +3. **A provider-specific over-generalization caught:** KD does **not** unlock voluptuous on + OpenAI (0/0/0, all 5 reps). The published "KD cracked voluptuous 0→39%" was **Sonnet-only** + and must be qualified to the provider. +4. **Calibration:** significance tracks effect size vs noise — large unlocks clear K=5, small + near-saturated effects (cachetools) correctly do not. This is the cost-effective design: + sample the large-effect claim-bearing cells; near-saturated cells need much larger K. + +## Caveats / scope +- **OpenAI only** (reliable on the run host); Anthropic-provider claims (incl. the Sonnet KD + voluptuous result) need a separate run when the provider is stable / off-peak. +- **K=5** — bootstrap CIs are wide on bimodal cells; raising K tightens them where needed. +- **reflexion** is scored at its *last* iteration (`score_branch`), vs the published *best* + iteration — may understate reflexion; refine if a reflexion claim is load-bearing. +- These are **2 libs** (a demonstration that the pipeline produces calibrated significance), + not yet the full set of cited cells. + +## Reproduce +```bash +# WSL: bash sync_to_wsl.sh && bash preflight_clean.sh +python baselines/sampling/repeat_runner.py --arch --provider openai --libs --reps 5 --out-dir +python baselines/sampling/stats_analyze.py --results-dir +``` diff --git a/commit0/baselines/sampling/repeat_runner.py b/commit0/baselines/sampling/repeat_runner.py index 0f4cd5d..0f6f0c6 100644 --- a/commit0/baselines/sampling/repeat_runner.py +++ b/commit0/baselines/sampling/repeat_runner.py @@ -38,6 +38,11 @@ def runner_cmd(arch: str, provider: str, lib: str) -> list[str]: if arch == "single_shot": script = "run_lite_single_shot.py" if provider == "anthropic" else "run_lite_single_shot_openai.py" return [py, str(BASELINES / script), "--only", lib] + if arch == "kaizen_delta": + return [py, str(BASELINES / "run_lite_kaizen_delta.py"), "--provider", provider, "--only", lib] + if arch == "reflexion": + script = "run_lite_reflexion.py" if provider == "anthropic" else "run_lite_reflexion_openai.py" + return [py, str(BASELINES / script), "--only", lib] raise ValueError(f"unsupported arch for sampling scaffold: {arch}") def branch_of(arch: str, provider: str) -> str: @@ -45,8 +50,26 @@ def branch_of(arch: str, provider: str) -> str: return f"{arch}_{provider}" # A1: per-provider branch (matches the fixed runners) if arch == "single_shot": return "single_shot_sonnet" if provider == "anthropic" else "single_shot_openai" + if arch == "reflexion": + return "reflexion_sonnet" if provider == "anthropic" else "reflexion_openai" raise ValueError(arch) +def _cost_from_runner_json(d: dict, provider: str) -> float: + """cost_usd if recorded (>0), else compute from tokens (single_shot records tokens, + not totals.cost_usd). Mirrors value_add_fingerprint pricing so the valid-rep gate + doesn't discard a real run as 'billed nothing'.""" + totals = d.get("totals") or {} + c = totals.get("cost_usd") + if c and float(c) > 0: + return float(c) + def g(key): + return float(totals.get(key, d.get(key, 0)) or 0) + inp, out, cached = g("input_tokens"), g("output_tokens"), g("cached_input_tokens") + if provider == "anthropic": + return (inp * 3 + out * 15) / 1_000_000 + return ((inp - cached) * 1.25 + cached * 0.125 + out * 10) / 1_000_000 + + def one_rep(arch, provider, lib, k, temperature, seed, dry, model=None, seed_key=None): """Run + score one rep. Returns the rep dict (or None on dry-run). @@ -71,12 +94,15 @@ def one_rep(arch, provider, lib, k, temperature, seed, dry, model=None, seed_key # robust re-score of the branch the agent just produced (score_branch, not the # runner's own scoring) so every rep is scored identically (T1-T4). sc = sb.score_branch(lib, branch_of(arch, provider)) - # cost from the runner's own JSON (it tracks the LLM spend) + # cost from the runner's own JSON. Some runners (single_shot) record TOKENS but not + # totals.cost_usd -> read cost_usd if present, else compute from tokens (mirrors + # value_add_fingerprint pricing). Otherwise a real run reads as $0 and the valid-rep + # gate (cost<=0) would wrongly discard it. runner_json = WORKSPACE / "baselines" / "results" / f"{lib}_{arch}_{provider}.json" cost = 0.0 if runner_json.exists(): try: - cost = float((json.loads(runner_json.read_text()).get("totals") or {}).get("cost_usd", 0) or 0) + cost = _cost_from_runner_json(json.loads(runner_json.read_text()), provider) except Exception: pass return { @@ -141,7 +167,8 @@ def _fix_jinja_editable_install(): def main(): ap = argparse.ArgumentParser() - ap.add_argument("--arch", required=True, choices=["aider", "smolagents", "single_shot"]) + ap.add_argument("--arch", required=True, + choices=["aider", "smolagents", "single_shot", "kaizen_delta", "reflexion"]) ap.add_argument("--provider", required=True, choices=["anthropic", "openai"]) ap.add_argument("--libs", nargs="+", required=True) ap.add_argument("--reps", type=int, default=5)