feat: cache the stable system prompt across runs, always-on no-overth…

evals/runner/harness.go

@@ -42,12 +42,30 @@ type Result struct {
 	CostUSD      float64 `json:"cost_usd"`
 	Error        string  `json:"error,omitempty"`       // harness or agent failure, not a test failure
 	TestOutput   string  `json:"test_output,omitempty"` // tail of go test output when it failed
+
+	// Set by compare mode, which meters spend at the API.
+	Agent            string   `json:"agent,omitempty"`
+	Run              int      `json:"run,omitempty"`
+	Requests         int      `json:"requests,omitempty"`
+	CacheReadTokens  int      `json:"cache_read_tokens,omitempty"`
+	CacheWriteTokens int      `json:"cache_write_tokens,omitempty"`
+	UnpricedModels   []string `json:"unpriced_models,omitempty"`
+	Skipped          bool     `json:"skipped,omitempty"` // never ran: interrupted or over budget
 }
 
 // Usage is what an agent run reports back about its spend.
 type Usage struct {
 	InputTokens  int
 	OutputTokens int
+
+	// Metered usage comes from the compare-mode proxy, which saw every API
+	// response the agent got; its CostUSD replaces the price-table estimate.
+	Metered          bool
+	CostUSD          float64
+	Requests         int
+	CacheReadTokens  int
+	CacheWriteTokens int
+	UnpricedModels   []string
 }
 
 // Agent performs a task in workdir. The real one shells out to gocode; the
@@ -95,8 +113,9 @@ func LoadTasks(root string) ([]Task, error) {
 // RunTask copies the fixture to a scratch directory, lets the agent work in
 // it, restores the visible tests, adds the hidden ones, and runs go test.
 // The scratch directory is removed unless keep is set.
-func RunTask(ctx context.Context, t Task, model string, agent Agent, keep bool) Result {
-	res := Result{Task: t.ID, Model: model}
+func RunTask(ctx context.Context, t Task, model string, agent Agent, keep bool) (res Result) {
+	// Named result: the deferred duration write below must reach the caller.
+	res = Result{Task: t.ID, Model: model}
 	start := time.Now()
 	defer func() { res.Duration = time.Since(start).Seconds() }()
 
@@ -121,7 +140,11 @@ func RunTask(ctx context.Context, t Task, model string, agent Agent, keep bool)
 	usage, agentErr := agent.Run(agentCtx, workdir, t.Prompt)
 	cancel()
 	res.InputTokens, res.OutputTokens = usage.InputTokens, usage.OutputTokens
-	if price, ok := apiclient.PriceForModel(model); ok {
+	res.Requests, res.CacheReadTokens, res.CacheWriteTokens = usage.Requests, usage.CacheReadTokens, usage.CacheWriteTokens
+	res.UnpricedModels = usage.UnpricedModels
+	if usage.Metered {
+		res.CostUSD = usage.CostUSD
+	} else if price, ok := apiclient.PriceForModel(model); ok {
 		res.CostUSD = (float64(usage.InputTokens)*price.Input + float64(usage.OutputTokens)*price.Output) / 1_000_000
 	}
 	if agentErr != nil {

evals/runner/main.go

@@ -9,6 +9,7 @@ import (
 	"os/exec"
 	"os/signal"
 	"path/filepath"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -24,13 +25,31 @@ func main() {
 	maxTurns := flag.Int("max-turns", 40, "agent loop iterations per task")
 	outDir := flag.String("out", "", "results directory; default evals/results in this checkout")
 	keep := flag.Bool("keep", false, "keep scratch directories for inspection")
+	var agents multiFlag
+	flag.Var(&agents, "agent", "compare agent CLIs on one model instead of models on gocode: gocode, opencode, crush or claurst, optionally name=/path/to/binary (repeatable)")
+	runs := flag.Int("runs", 1, "with -agent: sweeps over the task set")
+	budget := flag.Float64("budget", 10, "with -agent: stop sending API requests once metered spend reaches this many USD (0 = no limit)")
+	var agentPath multiFlag
+	flag.Var(&agentPath, "agent-path", "with -agent: directory to put first on the agents' PATH, e.g. one holding rg (repeatable)")
+	catalog := flag.String("models-catalog", "", "with -agent: a models.dev api.json for agents that read one offline (claurst)")
 	flag.Parse()
 
 	if len(models) == 0 {
 		fmt.Fprintln(os.Stderr, "usage: go run ./evals/runner -model <alias-or-id> [-model ...]")
+		fmt.Fprintln(os.Stderr, "       go run ./evals/runner -model <claude-model> -agent gocode -agent opencode ...")
 		fmt.Fprintln(os.Stderr, "This spends tokens: roughly 13 agent runs per model, each a small Go task.")
 		os.Exit(2)
 	}
+	var specs []agentSpec
+	if len(agents) > 0 {
+		if len(models) != 1 {
+			fatal("-agent compares agents on one model; pass exactly one -model")
+		}
+		var err error
+		if specs, err = parseAgentSpecs(agents); err != nil {
+			fatal("%v", err)
+		}
+	}
 
 	root, err := moduleRoot()
 	if err != nil {
@@ -46,14 +65,14 @@ func main() {
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer stop()
 
-	if *binary == "" {
+	needGocode := len(specs) == 0 || slices.ContainsFunc(specs, func(s agentSpec) bool { return s.Name == "gocode" && s.Binary == "" })
+	if *binary == "" && needGocode {
 		*binary, err = buildGocode(ctx, root)
 		if err != nil {
 			fatal("building gocode: %v", err)
 		}
 		defer os.RemoveAll(filepath.Dir(*binary))
 	}
-	version := gocodeVersion(ctx, *binary)
 
 	tasks, err := LoadTasks(*tasksDir)
 	if err != nil {
@@ -64,6 +83,24 @@ func main() {
 		fatal("no tasks selected")
 	}
 
+	if len(specs) > 0 {
+		for i := range specs {
+			if specs[i].Name == "gocode" && specs[i].Binary == "" {
+				specs[i].Binary = *binary
+			}
+		}
+		err := runCompare(ctx, compareConfig{
+			Model: models[0], Agents: specs, Tasks: tasks, Runs: max(*runs, 1),
+			Parallel: *parallel, MaxTurns: *maxTurns, BudgetUSD: *budget, OutDir: *outDir,
+			Keep: *keep, AgentPath: agentPath, Catalog: *catalog,
+		})
+		if err != nil {
+			fatal("compare: %v", err)
+		}
+		return
+	}
+	version := gocodeVersion(ctx, *binary)
+
 	fmt.Fprintf(os.Stderr, "gocode %s, %d tasks, %d model(s), parallel=%d\n", version, len(tasks), len(models), *parallel)
 	byModel := map[string][]Result{}
 	for _, model := range models {