diff --git a/docs/agent-loop-lessons.md b/docs/agent-loop-lessons.md new file mode 100644 index 0000000..06a2f22 --- /dev/null +++ b/docs/agent-loop-lessons.md @@ -0,0 +1,54 @@ +# Agent-loop lessons for the fix-arm (from COMPILOT) + +Notes mined from *Agentic Auto-Scheduling: An Experimental Study of LLM-Guided Loop +Optimization* (COMPILOT, Merouani et al., PACT 2025). COMPILOT puts an off-the-shelf LLM in a +closed loop with a compiler: the model proposes loop transformations, the compiler checks +legality + measures speedup, the typed outcome is fed back, repeat. Different domain, but the +*agent-loop mechanics* map directly onto our fix-arm (LLM proposes a fix → re-audit verifies → +typed feedback → revise). This captures what transfers so it isn't lost. + +## Where COMPILOT validates what we already do + +- **Delegate correctness to a deterministic verifier, never the LLM (RQ7).** COMPILOT has the LLM + emit a *transformation*, and the compiler's polyhedral analysis guarantees legality. When they + instead let the LLM rewrite code directly and verified by output-comparison, **17.6% of + "passing" schedules were actually wrong** under random inputs, at **5.3× the tokens**. Our + fix-arm does exactly the safe thing: the LLM rewrites a window, but the **real analyzers + re-audit** and we **reject any fix that introduces a new finding** — formal-ish verification, + not eyeballing. Strong external validation of the tier+re-audit design (`docs/fix-arm.md §4`). +- **The feedback loop is the whole game (RQ6).** With feedback, +23–40% over open-loop, and the + gap *widens* with iterations. Our `AiFixApplier` already runs a verify→revise loop. +- **Stopping + give-up.** COMPILOT caps iterations and records premature stops; we cap + `max_rounds` and record `ai-gave-up`. + +## Adopted now (this change) + +- **Accumulated attempt history, not just the last rejection.** Our LLM client is *stateless* + (`complete(system, user)` — no chat history), so across revise rounds the model only ever saw + the **most recent** rejection reason and could re-propose an identical failed fix. COMPILOT's + loop works because *every* prior outcome stays in the dialogue (illegal-rate drops from ~60% at + T=1 as negative feedback accumulates — RQ3). Fix: `_revise` now threads the full history of + rejected candidates + their specific reasons into each prompt ("do NOT repeat them; try a + DIFFERENT fix"). +- **Specific typed feedback.** "it introduced new findings" → now lists *which* (`rule@file:line`), + giving the model something to act on, like COMPILOT's per-failure reasons. + +## Menu for later (documented, not built) + +- **Best-of-K restarts (RQ1/RQ9).** Beyond the sequential revise chain, restart the dialogue from + scratch K times and keep the best — COMPILOT uses K=5 to escape local optima. For hard T4/OWN + findings, K independent attempts (fresh context) → take the first that re-audits clean. A + quality/cost knob; diminishing returns measured (their T=30/K=5). +- **Reasoning-first / analysis step (RQ10).** COMPILOT forces a CoT analysis before proposing and + a reasoning section per turn — "tangible benefits". We currently suppress reasoning ("Return + ONLY the corrected replacement") for clean parsing. Tradeoff: a `…` + preamble (our fence regex already ignores anything outside the ```csharp block) would add the + benefit *and* seed the deferred AI-explanation layer — at some token + parse-robustness cost. +- **Two-stage check: cheap pre-filter before the expensive re-audit.** COMPILOT runs a + compiler-independent validity check (syntax, valid identifiers, preconditions) *before* the + costly compiler. Our re-audit (a full analyzer pass on the stand) is the expensive step; a + lightweight pre-check on the candidate (parses? actually different? brace-balanced? sane line + delta?) could skip a re-audit round for obviously-bad candidates. +- **Measured iteration budget.** COMPILOT charted tokens vs iterations and chose T/K at the + diminishing-returns knee. Our `max_rounds=3` is a guess; worth measuring once we run against a + real local model + real re-audit cost. diff --git a/fix/fixarm/ai_fix.py b/fix/fixarm/ai_fix.py index e25969a..18a1470 100644 --- a/fix/fixarm/ai_fix.py +++ b/fix/fixarm/ai_fix.py @@ -42,7 +42,7 @@ def build_user(rel: str, lines: list[str], a: int, b: int, finding, feedback: st """Prompt body: the file, the finding, the numbered window [a, b) to replace, and (on a revise round) why the previous attempt was rejected.""" numbered = "".join(f"{i + 1:>5} {lines[i]}" for i in range(a, b)) - fb = f"\nYour previous attempt was rejected: {feedback}\nTry again.\n" if feedback else "" + fb = f"\n{feedback}\n" if feedback else "" return (f"File: {rel}\n" f"Finding at line {finding.line} [{finding.rule}]: {finding.message}\n" f"{fb}\nReplace lines {a + 1}..{b} (return only their corrected form):\n\n{numbered}") @@ -110,6 +110,23 @@ def _still_present(after, f, line_tol=0) -> bool: for g in after) +def _fmt_findings(fs, limit=5) -> str: + shown = ", ".join(f"{g.rule}@{g.basename}:{g.line}" for g in fs[:limit]) + return shown + (f" (+{len(fs) - limit} more)" if len(fs) > limit else "") + + +def _history_block(history) -> str: + """Render every prior REJECTED candidate + its specific reason. The LLM client is stateless + (no chat history), so this is the agent's only episodic memory across revise rounds — without + it the model re-proposes the same failing fix (COMPILOT RQ3/RQ6; docs/agent-loop-lessons.md).""" + if not history: + return "" + out = ["Previous attempts were REJECTED — do NOT repeat them; try a DIFFERENT fix:"] + for i, (code, reason) in enumerate(history, 1): + out.append(f"\n[attempt {i} — rejected: {reason}]\n{code.rstrip()}") + return "\n".join(out) + + class AiFixApplier: """For each finding, ask the LLM to rewrite a window around it; splice the reply back. Inherits dry-run/diff/re-audit/gate/rollback from the wrapper. With `reaudit` @@ -152,9 +169,9 @@ def _revise(self, workdir, path, rel, cur, a, b, f, skipped): if cand is None: skipped.append((f, "ai-no-change")) return cand - feedback = "" + history = [] # (rejected candidate text, specific reason) per round for _ in range(self.max_rounds): - cand = self._propose(rel, cur, a, b, f, feedback) + cand = self._propose(rel, cur, a, b, f, _history_block(history)) if cand is None: skipped.append((f, "ai-no-change")) return None @@ -166,10 +183,17 @@ def _revise(self, workdir, path, rel, cur, a, b, f, skipped): with open(path, "w", encoding="utf-8") as fh: fh.write("".join(cur)) introduced = diff_findings(self.before, after, self.line_tol)[1] - if not _still_present(after, f, self.line_tol) and not introduced: + still_present = _still_present(after, f, self.line_tol) + if not still_present and not introduced: return cand # verified this round - feedback = ("it introduced new findings" - if introduced else "the finding is still reported") + reasons = [] # both causes can hold at once — feed back both + if still_present: + reasons.append(f"the finding [{f.rule}] is still reported") + if introduced: + reasons.append(f"it introduced new findings: {_fmt_findings(introduced)}") + reason = "; ".join(reasons) + window = cand[a:len(cand) - (len(cur) - b)] # just the rejected replacement, + history.append(("".join(window), reason)) # NOT the whole spliced file skipped.append((f, "ai-gave-up")) return None diff --git a/fix/tests/test_ai_fix.py b/fix/tests/test_ai_fix.py index a25c447..5c61a2f 100644 --- a/fix/tests/test_ai_fix.py +++ b/fix/tests/test_ai_fix.py @@ -151,6 +151,37 @@ def reaudit(wd): # content-aware: finding present iff 'x = shutil.rmtree(d, ignore_errors=True) +def test_ai_revise_accumulates_history(): + # the stateless client only knows what we put in the prompt — so every prior rejected + # attempt must be threaded forward, else the model can re-propose the same failing fix. + d, _ = _tmp(SRC) + bad1 = "```csharp\n int x = Compute();\n x = 0; // v1\n```" + bad2 = "```csharp\n int x = Compute();\n x = 0; // v2\n```" + client = MockLlmClient([bad1, bad2, GOOD_REPLY]) + + def reaudit(wd): + body = open(os.path.join(wd, "Core", "Sample.cs"), encoding="utf-8").read() + return [FINDING] if "x = 0;" in body else [] + + try: + applier = AiFixApplier([FINDING], client, reaudit=reaudit, before=[FINDING], + max_rounds=3, ctx=4) + res = run_fix(before=[FINDING], workdir=d, rule=FINDING.rule, applier=applier, + reaudit=reaudit, tier_of=_review_tier) + assert res.status == OK, res.ledger() + third = client.calls[2][1] # round-3 prompt + assert "attempt 1" in third and "attempt 2" in third, third + assert "// v1" in third and "// v2" in third, third + # the reason must be attached to EACH prior attempt, not merely present once + reason = f"the finding [{FINDING.rule}] is still reported" + assert third.count(reason) == 2, third + # history stores only the rejected REPLACEMENT window, never the whole spliced + # file — else two rejected rounds bloat the prompt with unrelated lines (Codex P2). + assert "namespace Sts.Core" not in third, third + finally: + shutil.rmtree(d, ignore_errors=True) + + def test_ai_loop_gives_up_after_max_rounds(): d, _ = _tmp(SRC) bad = ("```csharp\n int x = Compute();\n x = 0; // nope\n```")