From 127f73a3a55ea3124601b6b98935a9eb9f2af8a2 Mon Sep 17 00:00:00 2001 From: adanamanya Date: Mon, 2 Nov 2026 20:59:05 +0700 Subject: [PATCH] feat: cache the stable system prompt across runs, always-on no-overthink rules, agent comparison evals System prompt - The run-specific environment (working directory, date, git branch and changed-file count) moves to the end of the system prompt, under "# Environment". On Anthropic the cache breakpoint now sits before that section, so runs in other directories, on other days or with other git state reuse one cached prefix instead of writing a new one. Other providers get the same text in one piece. - The no-overthink rules (github.com/AlleyBo55/no-overthink, 6591c77) are embedded and always on. Two lines that contradicted them changed: "be proactive, anticipate what the user did not ask" now says to carry the asked task through, and "briefly summarize what you did" now says to lead with the result. - System prompt: 2,331 -> 2,806 tokens (Anthropic count_tokens). Evals - New -agent mode compares agent CLIs (gocode, opencode, crush, claurst) on one Claude model: fresh HOME and git repo per run, one macOS sandbox profile for all, and cost metered by a local proxy from the usage the API returns. Agents only ever get throwaway keys. -runs, -budget, -agent-path, -models-catalog. - Fix: durations were always recorded as 0s (deferred write to an unnamed result). Measured on claude-sonnet-5-5, 15 tasks x 3 runs, all agents 45/45: $ per solved task: gocode 0.013, claurst 0.018, crush 0.039, opencode 0.041. gocode before this change: 0.025 (1 run); cache writes per sweep 92k -> 19-28k. --- evals/runner/compare.go | 804 +++++++++++++++++++++ evals/runner/harness.go | 29 +- evals/runner/main.go | 41 +- evals/runner/meter.go | 394 ++++++++++ evals/runner/meter_test.go | 142 ++++ internal/apiclient/anthropic_cache.go | 32 +- internal/apiclient/anthropic_cache_test.go | 28 + internal/apitypes/types.go | 7 + internal/repl/no_overthink.md | 17 + internal/repl/repl.go | 61 +- 10 files changed, 1519 insertions(+), 36 deletions(-) create mode 100644 evals/runner/compare.go create mode 100644 evals/runner/meter.go create mode 100644 evals/runner/meter_test.go create mode 100644 internal/repl/no_overthink.md diff --git a/evals/runner/compare.go b/evals/runner/compare.go new file mode 100644 index 0000000..fcb8dc5 --- /dev/null +++ b/evals/runner/compare.go @@ -0,0 +1,804 @@ +package main + +// Compare mode runs several agent CLIs on the same tasks with the same +// model. Every run gets a fresh HOME and a fresh git repo holding the +// fixture, runs under one macOS sandbox profile, and reaches the model only +// through the metering proxy, so pass rate, time and cost are measured the +// same way for every agent. + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strings" + "sync" + "syscall" + "time" + "unicode/utf8" + + "github.com/AlleyBo55/gocode/internal/apiclient" +) + +const sandboxExec = "/usr/bin/sandbox-exec" + +// knownAgents are the CLIs compare mode knows how to drive headless. +var knownAgents = []string{"gocode", "opencode", "crush", "claurst"} + +type agentSpec struct { + Name string + Binary string // empty for gocode means the binary this runner builds +} + +func parseAgentSpecs(values []string) ([]agentSpec, error) { + var specs []agentSpec + for _, v := range values { + name, bin, _ := strings.Cut(v, "=") + name = strings.TrimSpace(name) + if !slices.Contains(knownAgents, name) { + return nil, fmt.Errorf("unknown agent %q (known: %s)", name, strings.Join(knownAgents, ", ")) + } + if slices.ContainsFunc(specs, func(s agentSpec) bool { return s.Name == name }) { + return nil, fmt.Errorf("agent %q given twice", name) + } + if bin == "" && name != "gocode" { + p, err := exec.LookPath(name) + if err != nil { + return nil, fmt.Errorf("%s is not on PATH; pass -agent %s=/path/to/%s", name, name, name) + } + bin = p + } + if bin != "" { + abs, err := filepath.Abs(bin) + if err != nil { + return nil, err + } + bin = abs + } + specs = append(specs, agentSpec{Name: name, Binary: bin}) + } + return specs, nil +} + +type compareConfig struct { + Model string + Agents []agentSpec + Tasks []Task + Runs int + Parallel int + MaxTurns int + BudgetUSD float64 + OutDir string + Keep bool + AgentPath []string + Catalog string +} + +// CompareFile is what a comparison writes to its results directory. +type CompareFile struct { + Model string `json:"model"` + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` + Runs int `json:"runs"` + MaxTurns int `json:"max_turns"` + BudgetUSD float64 `json:"budget_usd"` + SpentUSD float64 `json:"spent_usd"` + Price anthropicPrice `json:"price_per_mtok"` + Agents []AgentInfo `json:"agents"` + SandboxProfile string `json:"sandbox_profile"` + Results []Result `json:"results"` +} + +type AgentInfo struct { + Name string `json:"name"` + Version string `json:"version"` + Command []string `json:"command"` +} + +type sharedDirs struct { + homes, gocache, gomodcache, gopath string +} + +type taskCtxKey struct{} +type runNumCtxKey struct{} + +func runCompare(ctx context.Context, cfg compareConfig) error { + if runtime.GOOS != "darwin" { + return errors.New("compare mode sandboxes each agent with macOS sandbox-exec, so it runs on macOS only") + } + if _, err := os.Stat(sandboxExec); err != nil { + return fmt.Errorf("%s not found: %w", sandboxExec, err) + } + key := os.Getenv("ANTHROPIC_API_KEY") + if key == "" { + return errors.New("compare mode needs ANTHROPIC_API_KEY; agents never see it, the metering proxy adds it to each request") + } + model := apiclient.ResolveModelAlias(cfg.Model) + price, priced := priceFor(model, 0) + if !strings.HasPrefix(model, "claude-") || !priced { + return fmt.Errorf("compare mode meters Anthropic's API and needs a priced claude-* model; got %q", model) + } + userHome, err := os.UserHomeDir() + if err != nil { + return err + } + if real, err := filepath.EvalSymlinks(userHome); err == nil { + userHome = real + } + + root, err := os.MkdirTemp("", "gocode-compare-") + if err != nil { + return err + } + if root, err = filepath.EvalSymlinks(root); err != nil { + return err + } + if cfg.Keep { + fmt.Fprintf(os.Stderr, "scratch root kept at %s\n", root) + } else { + defer os.RemoveAll(root) + } + shared := sharedDirs{ + homes: filepath.Join(root, "homes"), + gocache: filepath.Join(root, "gocache"), + gomodcache: filepath.Join(root, "gomodcache"), + gopath: filepath.Join(root, "gopath"), + } + for _, d := range []string{shared.homes, shared.gocache, shared.gomodcache, shared.gopath} { + if err := os.MkdirAll(d, 0o755); err != nil { + return err + } + } + + m, err := newMeter("https://api.anthropic.com", key, cfg.BudgetUSD) + if err != nil { + return err + } + defer m.Close() + + started := time.Now().UTC() + runDir := filepath.Join(cfg.OutDir, started.Format("20060102-150405")+"_compare_"+sanitize(model)) + logDir := filepath.Join(runDir, "logs") + if err := os.MkdirAll(logDir, 0o755); err != nil { + return err + } + + path, readable := agentPATH(cfg.AgentPath, cfg.Agents, userHome) + profile := sandboxProfile(userHome, readable) + catalog := cfg.Catalog + if catalog != "" { + if catalog, err = filepath.Abs(catalog); err != nil { + return err + } + } + + var agents []*cliAgent + for _, s := range cfg.Agents { + a := &cliAgent{ + Name: s.Name, Binary: s.Binary, Model: model, MaxTurns: cfg.MaxTurns, + meter: m, shared: shared, path: path, profile: profile, + logDir: logDir, catalog: catalog, keep: cfg.Keep, + } + a.Version = a.version() + fmt.Fprintf(os.Stderr, "%s: %s (%s)\n", a.Name, a.Version, a.Binary) + agents = append(agents, a) + } + + fmt.Fprintln(os.Stderr, "warming the shared Go build cache") + warmGoCache(ctx, cfg.Tasks, agents[0].baseEnv(filepath.Join(root, "warm-home"), "")) + + fmt.Fprintf(os.Stderr, "model %s, %d tasks, %d agent(s), %d run(s), parallel=%d per agent, budget $%.2f\n", + model, len(cfg.Tasks), len(agents), cfg.Runs, cfg.Parallel, cfg.BudgetUSD) + byAgent := map[string][]Result{} + for run := 1; run <= cfg.Runs && ctx.Err() == nil && !m.Exhausted(); run++ { + var wg sync.WaitGroup + var mu sync.Mutex + for _, a := range agents { + wg.Add(1) + go func(a *cliAgent) { + defer wg.Done() + rs := runAgentTasks(ctx, a, cfg.Tasks, run, cfg.Parallel, cfg.Keep, m) + mu.Lock() + byAgent[a.Name] = append(byAgent[a.Name], rs...) + mu.Unlock() + }(a) + } + wg.Wait() + } + + cf := CompareFile{ + Model: model, StartedAt: started, FinishedAt: time.Now().UTC(), + Runs: cfg.Runs, MaxTurns: cfg.MaxTurns, BudgetUSD: cfg.BudgetUSD, SpentUSD: m.Spent(), + Price: price, SandboxProfile: strings.ReplaceAll(profile, userHome, "~"), + } + for _, a := range agents { + cf.Agents = append(cf.Agents, AgentInfo{Name: a.Name, Version: a.Version, Command: a.args("", "")}) + cf.Results = append(cf.Results, byAgent[a.Name]...) + } + if err := writeJSONFile(filepath.Join(runDir, "results.json"), cf); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(runDir, "report.md"), []byte(renderComparison(cf, cfg.Tasks, true)), 0o644); err != nil { + return err + } + fmt.Print(renderComparison(cf, cfg.Tasks, false)) + fmt.Fprintf(os.Stderr, "results, report and per-run logs in %s\n", runDir) + return nil +} + +func runAgentTasks(ctx context.Context, a *cliAgent, tasks []Task, run, parallel int, keep bool, m *meter) []Result { + results := make([]Result, len(tasks)) + sem := make(chan struct{}, max(parallel, 1)) + var wg sync.WaitGroup + for i, t := range tasks { + wg.Add(1) + sem <- struct{}{} + go func(i int, t Task) { + defer wg.Done() + defer func() { <-sem }() + if ctx.Err() != nil || m.Exhausted() { + results[i] = Result{Task: t.ID, Model: a.Model, Agent: a.Name, Run: run, Skipped: true, Error: "skipped: interrupted or over budget"} + return + } + fmt.Fprintf(os.Stderr, "[%s run %d] %s: running\n", a.Name, run, t.ID) + tctx := context.WithValue(context.WithValue(ctx, taskCtxKey{}, t.ID), runNumCtxKey{}, run) + r := RunTask(tctx, t, a.Model, a, keep) + r.Agent, r.Run = a.Name, run + results[i] = r + mark := "FAIL" + if r.Passed { + mark = "pass" + } + if r.Error != "" { + mark += " (" + firstLine(r.Error, 120) + ")" + } + fmt.Fprintf(os.Stderr, "[%s run %d] %s: %s %.0fs $%.3f, $%.2f spent so far\n", a.Name, run, t.ID, mark, r.Duration, r.CostUSD, m.Spent()) + }(i, t) + } + wg.Wait() + return results +} + +// cliAgent drives one agent CLI in its headless mode. It implements Agent. +type cliAgent struct { + Name, Binary, Version, Model string + MaxTurns int + + meter *meter + shared sharedDirs + path string + profile string + logDir string + catalog string + keep bool +} + +func (a *cliAgent) Run(ctx context.Context, workdir, prompt string) (Usage, error) { + task, _ := ctx.Value(taskCtxKey{}).(string) + run, _ := ctx.Value(runNumCtxKey{}).(int) + home, err := a.newHome(fmt.Sprintf("%s-%s-r%d-", a.Name, task, run)) + if err != nil { + return Usage{}, err + } + if !a.keep { + defer os.RemoveAll(home) + } + if err := gitInit(workdir, a.baseEnv(home, "")); err != nil { + return Usage{}, err + } + + token := a.meter.newRun() + extra, err := a.setup(home, token) + if err != nil { + a.meter.finishRun(token) + return Usage{}, err + } + logPath := filepath.Join(a.logDir, fmt.Sprintf("%s_%s_run%d.log", a.Name, task, run)) + logFile, err := os.Create(logPath) + if err != nil { + a.meter.finishRun(token) + return Usage{}, err + } + + cmd := exec.CommandContext(ctx, sandboxExec, append([]string{"-p", a.profile, a.Binary}, a.args(workdir, prompt)...)...) + cmd.Dir = workdir + cmd.Env = append(a.baseEnv(home, token), extra...) + cmd.Stdout, cmd.Stderr = logFile, logFile + // Its own process group, so a timeout also takes down the language + // servers and shells the agent started. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) } + cmd.WaitDelay = 10 * time.Second + runErr := cmd.Run() + if cmd.Process != nil { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } + logFile.Close() + + mu := a.meter.finishRun(token) + usage := Usage{ + InputTokens: mu.InputTokens, OutputTokens: mu.OutputTokens, + Metered: true, CostUSD: mu.CostUSD, Requests: mu.Requests, + CacheReadTokens: mu.CacheRead, CacheWriteTokens: mu.CacheWrite, UnpricedModels: mu.Unpriced, + } + switch { + case ctx.Err() != nil: + return usage, errors.New("agent timed out") + case runErr != nil: + return usage, fmt.Errorf("%s exited: %v: %s", a.Name, runErr, lastLines(logPath, 3)) + case mu.Requests == 0: + return usage, fmt.Errorf("%s made no metered API calls (%d HTTP errors): %s", a.Name, mu.HTTPErrors, lastLines(logPath, 3)) + } + return usage, nil +} + +// args is the agent's headless command line: the task prompt, the model, +// and whatever flag the agent needs to run tools without asking. +func (a *cliAgent) args(workdir, prompt string) []string { + turns := fmt.Sprint(a.MaxTurns) + switch a.Name { + case "gocode": + return []string{"prompt", "--model", a.Model, "--max-turns", turns, "--no-stream", "--output-format", "json", prompt} + case "claurst": + return []string{"--print", "--model", a.Model, "--permission-mode", "bypass-permissions", "--max-turns", turns, "--cwd", workdir, prompt} + case "crush": + // crush run approves every tool call in its session by itself. + return []string{"run", "--quiet", "--model", "anthropic/" + a.Model, "--cwd", workdir, prompt} + case "opencode": + return []string{"run", "--model", "anthropic/" + a.Model, "--auto", prompt} + } + return nil +} + +// setup writes the agent's config into its fresh HOME, pointing it at the +// metering proxy with the run's key, and returns extra environment. +func (a *cliAgent) setup(home, token string) ([]string, error) { + base := a.meter.URL + switch a.Name { + case "gocode": + return []string{"ANTHROPIC_BASE_URL=" + base}, nil + case "claurst": + dir := filepath.Join(home, ".claurst") + err := writeJSONFile(filepath.Join(dir, "settings.json"), map[string]any{ + "version": 1, + "provider": "anthropic", + "hasCompletedOnboarding": true, + "providers": map[string]any{ + "anthropic": map[string]any{"api_key": token, "api_base": base, "enabled": true}, + }, + // Seatbelt profiles do not nest: inside ours, claurst's own shell + // sandbox fails every command with exit 71. It runs under the + // same outer profile as the other agents instead. + "sandbox": map[string]any{"mode": "off", "network": false, "unsandboxedPrefixes": []string{}}, + }) + env := []string{"CLAURST_HOME=" + dir, "ANTHROPIC_BASE_URL=" + base, "CLAURST_DISABLE_MODELS_FETCH=1"} + if a.catalog != "" { + env = append(env, "CLAURST_MODELS_PATH="+a.catalog) + } + return env, err + case "crush": + err := writeJSONFile(filepath.Join(home, ".config", "crush", "crush.json"), map[string]any{ + "providers": map[string]any{ + "anthropic": map[string]any{"api_key": token, "base_url": base}, + }, + "options": map[string]any{"disable_provider_auto_update": true, "disable_metrics": true}, + }) + return []string{"ANTHROPIC_BASE_URL=" + base, "CRUSH_DISABLE_METRICS=1", "CRUSH_DISABLE_PROVIDER_AUTO_UPDATE=1"}, err + case "opencode": + cfg, err := json.Marshal(map[string]any{ + "$schema": "https://opencode.ai/config.json", + "autoupdate": false, + "share": "disabled", + "provider": map[string]any{ + "anthropic": map[string]any{"options": map[string]any{"baseURL": base + "/v1", "apiKey": token}}, + }, + }) + return []string{ + "OPENCODE_CONFIG_CONTENT=" + string(cfg), + "ANTHROPIC_BASE_URL=" + base + "/v1", + "OPENCODE_DISABLE_AUTOUPDATE=1", + "OPENCODE_DISABLE_MODELS_FETCH=1", + "OPENCODE_DISABLE_LSP_DOWNLOAD=1", + }, err + } + return nil, fmt.Errorf("unknown agent %q", a.Name) +} + +// baseEnv is the whole environment an agent gets: nothing is inherited, so +// no real API key or shell setup leaks in. Go uses caches shared across runs +// under the scratch root, and never the network. +func (a *cliAgent) baseEnv(home, apiKey string) []string { + env := []string{ + "HOME=" + home, + "USER=" + os.Getenv("USER"), + "LOGNAME=" + os.Getenv("LOGNAME"), + "PATH=" + a.path, + "SHELL=/bin/zsh", + "TMPDIR=" + filepath.Join(home, "tmp"), + "LANG=en_US.UTF-8", + "TERM=dumb", + "NO_COLOR=1", + "XDG_CONFIG_HOME=" + filepath.Join(home, ".config"), + "XDG_DATA_HOME=" + filepath.Join(home, ".local", "share"), + "XDG_STATE_HOME=" + filepath.Join(home, ".local", "state"), + "XDG_CACHE_HOME=" + filepath.Join(home, ".cache"), + "GOCACHE=" + a.shared.gocache, + "GOMODCACHE=" + a.shared.gomodcache, + "GOPATH=" + a.shared.gopath, + "GOENV=off", + "GOTOOLCHAIN=local", + "GOPROXY=off", + "GOSUMDB=off", + "GOWORK=off", + "GOFLAGS=", + "DO_NOT_TRACK=1", + } + if apiKey != "" { + env = append(env, "ANTHROPIC_API_KEY="+apiKey) + } + return env +} + +func (a *cliAgent) newHome(prefix string) (string, error) { + home, err := os.MkdirTemp(a.shared.homes, prefix) + if err != nil { + return "", err + } + if err := os.MkdirAll(filepath.Join(home, "tmp"), 0o755); err != nil { + return "", err + } + gitconfig := "[user]\n\tname = eval\n\temail = eval@localhost\n[init]\n\tdefaultBranch = main\n" + return home, os.WriteFile(filepath.Join(home, ".gitconfig"), []byte(gitconfig), 0o644) +} + +func (a *cliAgent) version() string { + home, err := a.newHome(a.Name + "-version-") + if err != nil { + return "unknown" + } + defer os.RemoveAll(home) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, a.Binary, "--version") + cmd.Env = a.baseEnv(home, "") + out, err := cmd.Output() + if err != nil { + return "unknown" + } + return firstLine(strings.TrimSpace(string(out)), 80) +} + +// gitInit commits the fixture, so every agent starts in a repository as it +// would on a real project. +func gitInit(dir string, env []string) error { + for _, args := range [][]string{{"init", "-q"}, {"add", "-A"}, {"commit", "-q", "--no-verify", "-m", "fixture"}} { + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(slices.Clone(env), "GIT_CONFIG_NOSYSTEM=1") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("git %s: %v: %s", args[0], err, bytes.TrimSpace(out)) + } + } + return nil +} + +// agentPATH builds the agents' PATH from the extra directories plus wherever +// go, gopls, git and rg live, and lists the ones inside the user's home that +// the sandbox must leave readable. +func agentPATH(extra []string, specs []agentSpec, userHome string) (string, []string) { + var dirs []string + add := func(d string) { + if real, err := filepath.EvalSymlinks(d); err == nil { + d = real + } + if d != "" && !slices.Contains(dirs, d) { + dirs = append(dirs, d) + } + } + for _, d := range extra { + if abs, err := filepath.Abs(d); err == nil { + add(abs) + } + } + for _, tool := range []string{"go", "gopls", "git", "rg"} { + if p, err := exec.LookPath(tool); err == nil { + add(filepath.Dir(p)) + } + } + for _, d := range []string{"/usr/bin", "/bin", "/usr/sbin", "/sbin"} { + add(d) + } + var readable []string + keep := func(p string) { + if real, err := filepath.EvalSymlinks(p); err == nil { + p = real + } + if within(p, userHome) && !slices.Contains(readable, p) { + readable = append(readable, p) + } + } + for _, d := range dirs { + keep(d) + } + for _, s := range specs { + if s.Binary != "" { + keep(s.Binary) + } + } + if out, err := exec.Command("go", "env", "GOROOT").Output(); err == nil { + keep(strings.TrimSpace(string(out))) + } + return strings.Join(dirs, ":"), readable +} + +func within(p, root string) bool { + return p == root || strings.HasPrefix(p, root+string(filepath.Separator)) +} + +// sandboxProfile hides the user's home from the agent, for reads and writes, +// apart from the tool directories it needs, and blocks the network apart +// from loopback, where the metering proxy listens. The scratch workdir, the +// agent's HOME and the shared Go caches all live in the temp directory. +func sandboxProfile(userHome string, readable []string) string { + var b strings.Builder + b.WriteString("(version 1)\n(allow default)\n") + fmt.Fprintf(&b, "(deny file-write* (subpath %q))\n", userHome) + fmt.Fprintf(&b, "(deny file-read* (subpath %q))\n", userHome) + for _, p := range readable { + fmt.Fprintf(&b, "(allow file-read* (subpath %q))\n", p) + } + b.WriteString("(deny network-outbound)\n") + b.WriteString("(allow network-outbound (remote ip \"localhost:*\"))\n") + return b.String() +} + +// warmGoCache compiles every fixture once with the shared cache, with each +// task's test flags too, so no agent pays for building the standard library. +func warmGoCache(ctx context.Context, tasks []Task, env []string) { + for _, t := range tasks { + dir, err := os.MkdirTemp("", "gocode-compare-warm-") + if err != nil { + return + } + if copyTree(filepath.Join(t.Dir, "fixture"), dir, nil) == nil { + for _, extra := range [][]string{nil, t.TestArgs} { + args := append([]string{"test", "-count=1", "-run", "^$", "./..."}, extra...) + cmd := exec.CommandContext(ctx, "go", args...) + cmd.Dir, cmd.Env = dir, env + _ = cmd.Run() // an unfinished fixture may not compile; the stdlib still gets cached + } + } + os.RemoveAll(dir) + } +} + +func writeJSONFile(path string, v any) error { + data, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return os.WriteFile(path, append(data, '\n'), 0o644) +} + +func lastLines(path string, n int) string { + data, err := os.ReadFile(path) + if err != nil { + return "" + } + lines := strings.Split(tail(string(data), n), "\n") + for i, l := range lines { + lines[i] = firstLine(l, 200) + } + return strings.Join(lines, " | ") +} + +func firstLine(s string, limit int) string { + s, _, _ = strings.Cut(s, "\n") + if utf8.RuneCountInString(s) > limit { + s = string([]rune(s)[:limit]) + "..." + } + return s +} + +type agentStats struct { + Attempted, Passed, Errors, Skipped int + Cost float64 + Requests, In, CacheRead, CacheW int + Out int + Durations []float64 + Unpriced []string +} + +func statsFor(rs []Result) agentStats { + var s agentStats + for _, r := range rs { + if r.Skipped { + s.Skipped++ + continue + } + s.Attempted++ + if r.Passed { + s.Passed++ + } + if r.Error != "" { + s.Errors++ + } + s.Cost += r.CostUSD + s.Requests += r.Requests + s.In += r.InputTokens + s.CacheRead += r.CacheReadTokens + s.CacheW += r.CacheWriteTokens + s.Out += r.OutputTokens + s.Durations = append(s.Durations, r.Duration) + for _, m := range r.UnpricedModels { + if !slices.Contains(s.Unpriced, m) { + s.Unpriced = append(s.Unpriced, m) + } + } + } + return s +} + +func median(xs []float64) float64 { + if len(xs) == 0 { + return 0 + } + s := slices.Clone(xs) + slices.Sort(s) + n := len(s) + if n%2 == 1 { + return s[n/2] + } + return (s[n/2-1] + s[n/2]) / 2 +} + +func taskCell(rs []Result) string { + s := statsFor(rs) + if s.Attempted == 0 { + return "skipped" + } + mark := fmt.Sprintf("%d/%d", s.Passed, s.Attempted) + if s.Attempted == 1 { + mark = "✗" + if s.Passed == 1 { + mark = "✓" + } + } + cell := fmt.Sprintf("%s %.0fs $%.3f", mark, median(s.Durations), s.Cost/float64(s.Attempted)) + if s.Errors > 0 { + cell += " !" + } + return cell +} + +// renderComparison prints tasks down and agents across, then one summary +// row per measure, as aligned text or as a Markdown report. +func renderComparison(cf CompareFile, tasks []Task, markdown bool) string { + byAgent := map[string][]Result{} + for _, r := range cf.Results { + byAgent[r.Agent] = append(byAgent[r.Agent], r) + } + header := []string{"task"} + for _, a := range cf.Agents { + header = append(header, a.Name) + } + rows := [][]string{header} + for _, t := range tasks { + row := []string{t.ID} + for _, a := range cf.Agents { + var rs []Result + for _, r := range byAgent[a.Name] { + if r.Task == t.ID { + rs = append(rs, r) + } + } + row = append(row, taskCell(rs)) + } + rows = append(rows, row) + } + summary := func(label string, f func(s agentStats) string) { + row := []string{label} + for _, a := range cf.Agents { + row = append(row, f(statsFor(byAgent[a.Name]))) + } + rows = append(rows, row) + } + summary("pass rate", func(s agentStats) string { + if s.Attempted == 0 { + return "-" + } + return fmt.Sprintf("%d/%d (%.0f%%)", s.Passed, s.Attempted, 100*float64(s.Passed)/float64(s.Attempted)) + }) + summary("$ per solved task", func(s agentStats) string { + if s.Passed == 0 { + return "n/a" + } + return fmt.Sprintf("$%.3f", s.Cost/float64(s.Passed)) + }) + summary("total cost", func(s agentStats) string { return fmt.Sprintf("$%.2f", s.Cost) }) + summary("median time per task", func(s agentStats) string { return fmt.Sprintf("%.0fs", median(s.Durations)) }) + summary("API requests", func(s agentStats) string { return fmt.Sprint(s.Requests) }) + summary("tokens in / cache read / cache write / out", func(s agentStats) string { + return fmt.Sprintf("%s / %s / %s / %s", human(s.In), human(s.CacheRead), human(s.CacheW), human(s.Out)) + }) + summary("agent errors", func(s agentStats) string { + out := fmt.Sprint(s.Errors) + if s.Skipped > 0 { + out += fmt.Sprintf(", %d skipped", s.Skipped) + } + if len(s.Unpriced) > 0 { + out += ", unpriced: " + strings.Join(s.Unpriced, " ") + } + return out + }) + + var b strings.Builder + if markdown { + fmt.Fprintf(&b, "# Agent comparison on %s\n\n", cf.Model) + fmt.Fprintf(&b, "%s · %d tasks × %d run(s) · scored by hidden tests · cost metered at the API · $%.2f spent\n\n", + cf.StartedAt.Format("2006-01-02"), len(tasks), cf.Runs, cf.SpentUSD) + } else { + fmt.Fprintf(&b, "%s, %d tasks × %d run(s), $%.2f spent\n", cf.Model, len(tasks), cf.Runs, cf.SpentUSD) + } + b.WriteString(formatTable(rows, markdown)) + b.WriteString("\nEach cell: pass mark (or passes/runs), median wall time, mean cost per attempt. ! = the agent process failed or timed out; it was still scored on the files it left.\n") + if !markdown { + return b.String() + } + p := cf.Price + b.WriteString("\n## How this was measured\n\n") + for _, a := range cf.Agents { + fmt.Fprintf(&b, "- **%s** %s: `%s`\n", a.Name, a.Version, strings.Join(a.Command, " ")) + } + fmt.Fprintf(&b, "- One model for every agent: `%s`, each agent at its own defaults otherwise. Turn caps: %d where the agent has a flag for it; every task has a %s wall-clock limit from its task.json.\n", + cf.Model, cf.MaxTurns, "300 s") + b.WriteString("- Every run starts from a fresh HOME and a fresh git repo holding the fixture, with full tool auto-approval, under one macOS sandbox profile for all agents: no reads or writes under the user's home except the Go tool directories, no network except loopback.\n") + b.WriteString("- Scoring: visible tests are restored after the agent finishes, hidden tests it never saw are added, then `go test ./...` (with `-race` where the task asks) decides pass or fail.\n") + fmt.Fprintf(&b, "- Cost: every API request each agent made, including title, summary and sub-agent calls, passed through one local proxy that read the usage the API returned and priced it at Anthropic list price (per million tokens for %s: input $%.2f, output $%.2f, 5-minute cache write $%.2f, 1-hour cache write $%.2f, cache read $%.2f). Agents never saw the real key.\n", + cf.Model, p.Input, p.Output, p.CacheWrite5m, p.CacheWrite1h, p.CacheRead) + b.WriteString("- claurst's own shell sandbox was off, because macOS sandbox profiles do not nest; it ran under the same outer profile as the others.\n") + b.WriteString("- Fifteen small Go tasks is a smoke test, not a benchmark. Re-run with `-runs 3` before trusting a gap of one or two tasks.\n") + return b.String() +} + +func formatTable(rows [][]string, markdown bool) string { + var b strings.Builder + if markdown { + for i, r := range rows { + b.WriteString("| " + strings.Join(r, " | ") + " |\n") + if i == 0 { + seps := make([]string, len(r)) + for j := range seps { + seps[j] = "---" + } + b.WriteString("|" + strings.Join(seps, "|") + "|\n") + } + } + return b.String() + } + widths := make([]int, len(rows[0])) + for _, r := range rows { + for j, c := range r { + widths[j] = max(widths[j], utf8.RuneCountInString(c)) + } + } + for _, r := range rows { + for j, c := range r { + if j > 0 { + b.WriteString(" ") + } + b.WriteString(c + strings.Repeat(" ", widths[j]-utf8.RuneCountInString(c))) + } + b.WriteString("\n") + } + return b.String() +} diff --git a/evals/runner/harness.go b/evals/runner/harness.go index 8cc3bf8..3249efe 100644 --- a/evals/runner/harness.go +++ b/evals/runner/harness.go @@ -42,12 +42,30 @@ type Result struct { CostUSD float64 `json:"cost_usd"` Error string `json:"error,omitempty"` // harness or agent failure, not a test failure TestOutput string `json:"test_output,omitempty"` // tail of go test output when it failed + + // Set by compare mode, which meters spend at the API. + Agent string `json:"agent,omitempty"` + Run int `json:"run,omitempty"` + Requests int `json:"requests,omitempty"` + CacheReadTokens int `json:"cache_read_tokens,omitempty"` + CacheWriteTokens int `json:"cache_write_tokens,omitempty"` + UnpricedModels []string `json:"unpriced_models,omitempty"` + Skipped bool `json:"skipped,omitempty"` // never ran: interrupted or over budget } // Usage is what an agent run reports back about its spend. type Usage struct { InputTokens int OutputTokens int + + // Metered usage comes from the compare-mode proxy, which saw every API + // response the agent got; its CostUSD replaces the price-table estimate. + Metered bool + CostUSD float64 + Requests int + CacheReadTokens int + CacheWriteTokens int + UnpricedModels []string } // Agent performs a task in workdir. The real one shells out to gocode; the @@ -95,8 +113,9 @@ func LoadTasks(root string) ([]Task, error) { // RunTask copies the fixture to a scratch directory, lets the agent work in // it, restores the visible tests, adds the hidden ones, and runs go test. // The scratch directory is removed unless keep is set. -func RunTask(ctx context.Context, t Task, model string, agent Agent, keep bool) Result { - res := Result{Task: t.ID, Model: model} +func RunTask(ctx context.Context, t Task, model string, agent Agent, keep bool) (res Result) { + // Named result: the deferred duration write below must reach the caller. + res = Result{Task: t.ID, Model: model} start := time.Now() defer func() { res.Duration = time.Since(start).Seconds() }() @@ -121,7 +140,11 @@ func RunTask(ctx context.Context, t Task, model string, agent Agent, keep bool) usage, agentErr := agent.Run(agentCtx, workdir, t.Prompt) cancel() res.InputTokens, res.OutputTokens = usage.InputTokens, usage.OutputTokens - if price, ok := apiclient.PriceForModel(model); ok { + res.Requests, res.CacheReadTokens, res.CacheWriteTokens = usage.Requests, usage.CacheReadTokens, usage.CacheWriteTokens + res.UnpricedModels = usage.UnpricedModels + if usage.Metered { + res.CostUSD = usage.CostUSD + } else if price, ok := apiclient.PriceForModel(model); ok { res.CostUSD = (float64(usage.InputTokens)*price.Input + float64(usage.OutputTokens)*price.Output) / 1_000_000 } if agentErr != nil { diff --git a/evals/runner/main.go b/evals/runner/main.go index 761d858..6c6b256 100644 --- a/evals/runner/main.go +++ b/evals/runner/main.go @@ -9,6 +9,7 @@ import ( "os/exec" "os/signal" "path/filepath" + "slices" "strings" "sync" "time" @@ -24,13 +25,31 @@ func main() { maxTurns := flag.Int("max-turns", 40, "agent loop iterations per task") outDir := flag.String("out", "", "results directory; default evals/results in this checkout") keep := flag.Bool("keep", false, "keep scratch directories for inspection") + var agents multiFlag + flag.Var(&agents, "agent", "compare agent CLIs on one model instead of models on gocode: gocode, opencode, crush or claurst, optionally name=/path/to/binary (repeatable)") + runs := flag.Int("runs", 1, "with -agent: sweeps over the task set") + budget := flag.Float64("budget", 10, "with -agent: stop sending API requests once metered spend reaches this many USD (0 = no limit)") + var agentPath multiFlag + flag.Var(&agentPath, "agent-path", "with -agent: directory to put first on the agents' PATH, e.g. one holding rg (repeatable)") + catalog := flag.String("models-catalog", "", "with -agent: a models.dev api.json for agents that read one offline (claurst)") flag.Parse() if len(models) == 0 { fmt.Fprintln(os.Stderr, "usage: go run ./evals/runner -model [-model ...]") + fmt.Fprintln(os.Stderr, " go run ./evals/runner -model -agent gocode -agent opencode ...") fmt.Fprintln(os.Stderr, "This spends tokens: roughly 13 agent runs per model, each a small Go task.") os.Exit(2) } + var specs []agentSpec + if len(agents) > 0 { + if len(models) != 1 { + fatal("-agent compares agents on one model; pass exactly one -model") + } + var err error + if specs, err = parseAgentSpecs(agents); err != nil { + fatal("%v", err) + } + } root, err := moduleRoot() if err != nil { @@ -46,14 +65,14 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() - if *binary == "" { + needGocode := len(specs) == 0 || slices.ContainsFunc(specs, func(s agentSpec) bool { return s.Name == "gocode" && s.Binary == "" }) + if *binary == "" && needGocode { *binary, err = buildGocode(ctx, root) if err != nil { fatal("building gocode: %v", err) } defer os.RemoveAll(filepath.Dir(*binary)) } - version := gocodeVersion(ctx, *binary) tasks, err := LoadTasks(*tasksDir) if err != nil { @@ -64,6 +83,24 @@ func main() { fatal("no tasks selected") } + if len(specs) > 0 { + for i := range specs { + if specs[i].Name == "gocode" && specs[i].Binary == "" { + specs[i].Binary = *binary + } + } + err := runCompare(ctx, compareConfig{ + Model: models[0], Agents: specs, Tasks: tasks, Runs: max(*runs, 1), + Parallel: *parallel, MaxTurns: *maxTurns, BudgetUSD: *budget, OutDir: *outDir, + Keep: *keep, AgentPath: agentPath, Catalog: *catalog, + }) + if err != nil { + fatal("compare: %v", err) + } + return + } + version := gocodeVersion(ctx, *binary) + fmt.Fprintf(os.Stderr, "gocode %s, %d tasks, %d model(s), parallel=%d\n", version, len(tasks), len(models), *parallel) byModel := map[string][]Result{} for _, model := range models { diff --git a/evals/runner/meter.go b/evals/runner/meter.go new file mode 100644 index 0000000..f51d356 --- /dev/null +++ b/evals/runner/meter.go @@ -0,0 +1,394 @@ +package main + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httputil" + "net/url" + "slices" + "strings" + "sync" + "time" +) + +// Compare mode bills every agent through this proxy. Each agent run gets a +// throwaway API key; the proxy swaps it for the real key, forwards the +// request to Anthropic, and reads the usage the API reports in the response. +// The cost column is the API's own token counts at list price: one referee +// for every agent, whatever each CLI estimates for itself. + +// anthropicPrice is a model's list price in USD per million tokens, split +// the way the Messages API reports usage. +type anthropicPrice struct { + Input float64 `json:"input"` + Output float64 `json:"output"` + CacheWrite5m float64 `json:"cache_write_5m"` + CacheWrite1h float64 `json:"cache_write_1h"` + CacheRead float64 `json:"cache_read"` +} + +// anthropicPrices are Anthropic's list prices from +// platform.claude.com/docs/en/about-claude/pricing, checked 2026-11-02. +// Keys match model ids by longest prefix, so dated ids price correctly. +var anthropicPrices = map[string]anthropicPrice{ + "claude-fable-5-1": {10, 50, 12.50, 20, 0.25}, + "claude-opus-5-5": {4, 20, 5, 8, 0.20}, + "claude-sonnet-5-5": {2, 10, 2.50, 4, 0.10}, + "claude-haiku-5-5": {0.10, 0.50, 0.125, 0.20, 0.01}, + "claude-fable-5": {10, 50, 12.50, 20, 1}, + "claude-opus-5": {5, 25, 6.25, 10, 0.50}, + "claude-sonnet-5": {2, 10, 2.50, 4, 0.20}, + "claude-opus-4-8": {5, 25, 6.25, 10, 0.50}, + "claude-opus-4-7": {5, 25, 6.25, 10, 0.50}, + "claude-opus-4-6": {5, 25, 6.25, 10, 0.50}, + "claude-opus-4-5": {5, 25, 6.25, 10, 0.50}, + "claude-sonnet-4-6": {3, 15, 3.75, 6, 0.30}, + "claude-sonnet-4-5": {3, 15, 3.75, 6, 0.30}, + "claude-haiku-4-5": {1, 5, 1.25, 2, 0.10}, +} + +// Haiku 5.5 is priced by prompt length: a request whose prompt (input plus +// cache reads and writes) is over 100,000 tokens pays these rates instead. +var haiku55LongPrompt = anthropicPrice{0.50, 2.50, 0.625, 1, 0.05} + +func priceFor(model string, promptTokens int) (anthropicPrice, bool) { + id := strings.ToLower(strings.TrimSpace(model)) + best := "" + for k := range anthropicPrices { + if strings.HasPrefix(id, k) && len(k) > len(best) { + best = k + } + } + if best == "" { + return anthropicPrice{}, false + } + if best == "claude-haiku-5-5" && promptTokens > 100_000 { + return haiku55LongPrompt, true + } + return anthropicPrices[best], true +} + +// apiUsage is the usage object of a Messages API response. +type apiUsage struct { + InputTokens int `json:"input_tokens"` + CacheCreationInputTokens int `json:"cache_creation_input_tokens"` + CacheReadInputTokens int `json:"cache_read_input_tokens"` + OutputTokens int `json:"output_tokens"` + CacheCreation struct { + Ephemeral5m int `json:"ephemeral_5m_input_tokens"` + Ephemeral1h int `json:"ephemeral_1h_input_tokens"` + } `json:"cache_creation"` +} + +// merge folds a later usage report into u. Streamed counts are cumulative +// and a message_delta may omit fields message_start carried, so each counter +// keeps the larger value. +func (u *apiUsage) merge(o apiUsage) { + u.InputTokens = max(u.InputTokens, o.InputTokens) + u.CacheCreationInputTokens = max(u.CacheCreationInputTokens, o.CacheCreationInputTokens) + u.CacheReadInputTokens = max(u.CacheReadInputTokens, o.CacheReadInputTokens) + u.OutputTokens = max(u.OutputTokens, o.OutputTokens) + u.CacheCreation.Ephemeral5m = max(u.CacheCreation.Ephemeral5m, o.CacheCreation.Ephemeral5m) + u.CacheCreation.Ephemeral1h = max(u.CacheCreation.Ephemeral1h, o.CacheCreation.Ephemeral1h) +} + +// cacheWrites splits cache-write tokens by TTL. Without the breakdown every +// write is priced at the default 5-minute TTL. +func (u apiUsage) cacheWrites() (fiveMin, oneHour int) { + if b := u.CacheCreation; b.Ephemeral5m+b.Ephemeral1h > 0 { + return b.Ephemeral5m, b.Ephemeral1h + } + return u.CacheCreationInputTokens, 0 +} + +func costOf(model string, u apiUsage) (float64, bool) { + w5, w1 := u.cacheWrites() + p, ok := priceFor(model, u.InputTokens+w5+w1+u.CacheReadInputTokens) + if !ok { + return 0, false + } + return (float64(u.InputTokens)*p.Input + + float64(w5)*p.CacheWrite5m + + float64(w1)*p.CacheWrite1h + + float64(u.CacheReadInputTokens)*p.CacheRead + + float64(u.OutputTokens)*p.Output) / 1_000_000, true +} + +// MeteredUsage is what the proxy saw for one agent run. +type MeteredUsage struct { + Requests int + InputTokens int // uncached input + CacheWrite int + CacheRead int + OutputTokens int + CostUSD float64 + Unpriced []string // models the price table does not know + HTTPErrors int // non-2xx responses from the API +} + +type meter struct { + upstream *url.URL + realKey string + budget float64 + proxy *httputil.ReverseProxy + srv *http.Server + URL string // http://127.0.0.1:port, what agents use as their base URL + + mu sync.Mutex + runs map[string]*MeteredUsage + spent float64 +} + +type runCtxKey struct{} + +// newMeter starts the proxy on a loopback port. A budget of 0 is unlimited. +func newMeter(upstream, realKey string, budget float64) (*meter, error) { + target, err := url.Parse(upstream) + if err != nil { + return nil, err + } + m := &meter{upstream: target, realKey: realKey, budget: budget, runs: map[string]*MeteredUsage{}} + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxIdleConnsPerHost = 32 + m.proxy = &httputil.ReverseProxy{ + Rewrite: func(pr *httputil.ProxyRequest) { + pr.SetURL(m.upstream) + pr.Out.Header.Del("Authorization") + pr.Out.Header.Set("X-Api-Key", m.realKey) + // With no Accept-Encoding of its own the transport asks for + // gzip and decompresses, so the usage below stays readable. + pr.Out.Header.Del("Accept-Encoding") + }, + ModifyResponse: m.meterResponse, + Transport: transport, + FlushInterval: -1, + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + m.URL = "http://" + ln.Addr().String() + m.srv = &http.Server{Handler: m, ReadHeaderTimeout: 30 * time.Second} + go func() { _ = m.srv.Serve(ln) }() + return m, nil +} + +func (m *meter) Close() error { return m.srv.Close() } + +// newRun registers a run and returns the API key its agent should use. +func (m *meter) newRun() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + token := "sk-ant-api03-eval-" + hex.EncodeToString(b) + m.mu.Lock() + m.runs[token] = &MeteredUsage{} + m.mu.Unlock() + return token +} + +// finishRun retires a run's key and returns what the run spent. Requests +// that arrive later with that key are refused. +func (m *meter) finishRun(token string) MeteredUsage { + m.mu.Lock() + defer m.mu.Unlock() + u := m.runs[token] + delete(m.runs, token) + if u == nil { + return MeteredUsage{} + } + out := *u + out.Unpriced = slices.Clone(u.Unpriced) + slices.Sort(out.Unpriced) + return out +} + +func (m *meter) Spent() float64 { + m.mu.Lock() + defer m.mu.Unlock() + return m.spent +} + +func (m *meter) Exhausted() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.budget > 0 && m.spent >= m.budget +} + +func (m *meter) ServeHTTP(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("X-Api-Key") + if token == "" { + token = strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") + } + m.mu.Lock() + run := m.runs[token] + exhausted := m.budget > 0 && m.spent >= m.budget + m.mu.Unlock() + switch { + case run == nil: + writeAPIError(w, http.StatusUnauthorized, "authentication_error", "unknown or finished eval run key") + return + case exhausted: + writeAPIError(w, http.StatusBadRequest, "invalid_request_error", + fmt.Sprintf("the eval budget of $%.2f is spent; the comparison stopped sending requests", m.budget)) + return + } + // Some SDKs take a base URL ending in /v1 and others append it, so an + // agent configured either way reaches the same endpoint. + for strings.HasPrefix(r.URL.Path, "/v1/v1/") { + r.URL.Path = strings.TrimPrefix(r.URL.Path, "/v1") + } + r.URL.RawPath = "" + m.proxy.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), runCtxKey{}, run))) +} + +func writeAPIError(w http.ResponseWriter, status int, kind, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]any{ + "type": "error", + "error": map[string]string{"type": kind, "message": msg}, + }) +} + +func (m *meter) meterResponse(resp *http.Response) error { + run, _ := resp.Request.Context().Value(runCtxKey{}).(*MeteredUsage) + if run == nil { + return nil + } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + m.mu.Lock() + run.HTTPErrors++ + m.mu.Unlock() + return nil + } + resp.Body = &meterBody{ + ReadCloser: resp.Body, + sse: strings.HasPrefix(resp.Header.Get("Content-Type"), "text/event-stream"), + done: func(model string, u apiUsage) { m.record(run, model, u) }, + } + return nil +} + +func (m *meter) record(run *MeteredUsage, model string, u apiUsage) { + cost, priced := costOf(model, u) + w5, w1 := u.cacheWrites() + m.mu.Lock() + defer m.mu.Unlock() + run.Requests++ + run.InputTokens += u.InputTokens + run.CacheWrite += w5 + w1 + run.CacheRead += u.CacheReadInputTokens + run.OutputTokens += u.OutputTokens + run.CostUSD += cost + m.spent += cost + if !priced && !slices.Contains(run.Unpriced, model) { + run.Unpriced = append(run.Unpriced, model) + } +} + +// meterBody passes a response through unchanged while reading its usage: +// from the message_start and message_delta events when streamed, from the +// JSON body otherwise. Responses without usage (count_tokens, model lists) +// record nothing. +type meterBody struct { + io.ReadCloser + sse bool + buf []byte + model string + usage apiUsage + seen bool + done func(model string, u apiUsage) + once sync.Once +} + +const maxMeteredJSON = 16 << 20 + +func (b *meterBody) Read(p []byte) (int, error) { + n, err := b.ReadCloser.Read(p) + if n > 0 { + b.feed(p[:n]) + } + if err != nil { + b.finish() + } + return n, err +} + +func (b *meterBody) Close() error { + b.finish() + return b.ReadCloser.Close() +} + +func (b *meterBody) feed(p []byte) { + if !b.sse { + if len(b.buf)+len(p) <= maxMeteredJSON { + b.buf = append(b.buf, p...) + } + return + } + b.buf = append(b.buf, p...) + for { + i := bytes.IndexByte(b.buf, '\n') + if i < 0 { + break + } + b.event(b.buf[:i]) + b.buf = b.buf[i+1:] + } +} + +func (b *meterBody) event(line []byte) { + data, ok := bytes.CutPrefix(bytes.TrimRight(line, "\r"), []byte("data:")) + if !ok { + return + } + var ev struct { + Type string `json:"type"` + Message struct { + Model string `json:"model"` + Usage apiUsage `json:"usage"` + } `json:"message"` + Usage apiUsage `json:"usage"` + } + if json.Unmarshal(data, &ev) != nil { + return + } + switch ev.Type { + case "message_start": + b.model = ev.Message.Model + b.usage.merge(ev.Message.Usage) + b.seen = true + case "message_delta": + b.usage.merge(ev.Usage) + b.seen = true + } +} + +func (b *meterBody) finish() { + b.once.Do(func() { + if b.sse { + if len(b.buf) > 0 { + b.event(b.buf) + } + } else { + var body struct { + Model string `json:"model"` + Usage *apiUsage `json:"usage"` + } + if json.Unmarshal(b.buf, &body) == nil && body.Usage != nil { + b.model, b.usage, b.seen = body.Model, *body.Usage, true + } + } + b.buf = nil + if b.seen { + b.done(b.model, b.usage) + } + }) +} diff --git a/evals/runner/meter_test.go b/evals/runner/meter_test.go new file mode 100644 index 0000000..ae0c78a --- /dev/null +++ b/evals/runner/meter_test.go @@ -0,0 +1,142 @@ +package main + +import ( + "io" + "math" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func postTo(t *testing.T, base, key, path, body string) (*http.Response, string) { + t.Helper() + req, err := http.NewRequest("POST", base+path, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("x-api-key", key) + req.Header.Set("content-type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return resp, string(data) +} + +func TestMeterSwapsRunKeyForRealKeyAndRefusesUnknownKeys(t *testing.T) { + var gotKey, gotAuth, gotPath string + calls := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + gotKey, gotAuth, gotPath = r.Header.Get("X-Api-Key"), r.Header.Get("Authorization"), r.URL.Path + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"model":"claude-sonnet-5-5","usage":{"input_tokens":1000,"output_tokens":100}}`) + })) + defer upstream.Close() + m, err := newMeter(upstream.URL, "sk-real-key", 0) + if err != nil { + t.Fatal(err) + } + defer m.Close() + + token := m.newRun() + // A base URL configured with /v1 and an SDK that appends /v1 again + // must still reach /v1/messages. + resp, _ := postTo(t, m.URL, token, "/v1/v1/messages", `{}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d", resp.StatusCode) + } + if gotKey != "sk-real-key" || gotAuth != "" || gotPath != "/v1/messages" { + t.Errorf("upstream saw key=%q auth=%q path=%q; want the real key, no Authorization, /v1/messages", gotKey, gotAuth, gotPath) + } + + for _, key := range []string{"sk-ant-api03-guess", ""} { + resp, _ := postTo(t, m.URL, key, "/v1/messages", `{}`) + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("key %q: status = %d, want 401", key, resp.StatusCode) + } + } + used := m.finishRun(token) + if resp, _ := postTo(t, m.URL, token, "/v1/messages", `{}`); resp.StatusCode != http.StatusUnauthorized { + t.Errorf("a finished run's key still works: status %d", resp.StatusCode) + } + if calls != 1 { + t.Errorf("upstream called %d times; refused requests must never be forwarded", calls) + } + if want := (1000*2.0 + 100*10.0) / 1e6; used.Requests != 1 || math.Abs(used.CostUSD-want) > 1e-12 { + t.Errorf("usage = %+v, want 1 request costing %v", used, want) + } +} + +func TestMeterPricesStreamedUsageWithCacheTiers(t *testing.T) { + stream := "event: message_start\n" + + `data: {"type":"message_start","message":{"model":"claude-sonnet-5-5","usage":{"input_tokens":10,"cache_creation_input_tokens":1000,"cache_read_input_tokens":5000,"cache_creation":{"ephemeral_5m_input_tokens":600,"ephemeral_1h_input_tokens":400},"output_tokens":1}}}` + "\n\n" + + "event: content_block_delta\n" + + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hi"}}` + "\n\n" + + "event: message_delta\n" + + `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":200}}` + "\n\n" + + "event: message_stop\n" + + `data: {"type":"message_stop"}` + "\n\n" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, stream) + })) + defer upstream.Close() + m, err := newMeter(upstream.URL, "sk-real-key", 0) + if err != nil { + t.Fatal(err) + } + defer m.Close() + + token := m.newRun() + _, body := postTo(t, m.URL, token, "/v1/messages", `{"stream":true}`) + if body != stream { + t.Errorf("the agent must receive the stream byte for byte; got:\n%s", body) + } + u := m.finishRun(token) + // $2 input, $2.50 5-minute write, $4 1-hour write, $0.10 read, $10 output. + want := (10*2.0 + 600*2.5 + 400*4.0 + 5000*0.10 + 200*10.0) / 1e6 + if math.Abs(u.CostUSD-want) > 1e-12 { + t.Errorf("cost = %v, want %v", u.CostUSD, want) + } + if u.Requests != 1 || u.InputTokens != 10 || u.CacheWrite != 1000 || u.CacheRead != 5000 || u.OutputTokens != 200 { + t.Errorf("usage = %+v", u) + } +} + +func TestMeterStopsForwardingOnceTheBudgetIsSpent(t *testing.T) { + calls := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + // 1M output tokens on Sonnet 5.5 is $10. + _, _ = io.WriteString(w, `{"model":"claude-sonnet-5-5","usage":{"input_tokens":0,"output_tokens":1000000}}`) + })) + defer upstream.Close() + m, err := newMeter(upstream.URL, "sk-real-key", 5) + if err != nil { + t.Fatal(err) + } + defer m.Close() + + token := m.newRun() + if resp, _ := postTo(t, m.URL, token, "/v1/messages", `{}`); resp.StatusCode != http.StatusOK { + t.Fatalf("first request: status %d", resp.StatusCode) + } + if !m.Exhausted() { + t.Fatalf("spent $%.2f of a $5 budget but not exhausted", m.Spent()) + } + resp, body := postTo(t, m.URL, token, "/v1/messages", `{}`) + if resp.StatusCode != http.StatusBadRequest || !strings.Contains(body, "budget") { + t.Errorf("over budget: status %d body %s; want a 400 naming the budget", resp.StatusCode, body) + } + if calls != 1 { + t.Errorf("upstream called %d times after the budget was spent", calls) + } +} diff --git a/internal/apiclient/anthropic_cache.go b/internal/apiclient/anthropic_cache.go index 83a9e76..e551bbd 100644 --- a/internal/apiclient/anthropic_cache.go +++ b/internal/apiclient/anthropic_cache.go @@ -1,6 +1,8 @@ package apiclient import ( + "strings" + "github.com/AlleyBo55/gocode/internal/apitypes" ) @@ -14,6 +16,8 @@ import ( // // 1. On the system prompt. The cache prefix is ordered tools, then system, // then messages, so a breakpoint here covers the tool schemas as well. +// It sits before the run-specific environment section, so other runs +// (another directory, another day) hit the same cache entry. // 2. On the last block of the last message. Everything before the newest // user turn was already sent last turn and is a cache hit. // @@ -55,11 +59,7 @@ func anthropicWireRequest(req apitypes.MessageRequest, cache bool) anthropicRequ Stream: req.Stream, } if req.System != "" { - block := apitypes.InputContentBlock{Kind: "text", Text: req.System} - if cache { - block.CacheControl = apitypes.EphemeralCache() - } - out.System = []apitypes.InputContentBlock{block} + out.System = systemBlocks(req.System, cache) } if cache && len(req.Messages) > 0 { out.Messages = withTrailingBreakpoint(req.Messages) @@ -67,6 +67,28 @@ func anthropicWireRequest(req apitypes.MessageRequest, cache bool) anthropicRequ return out } +// systemBlocks turns the system prompt into wire blocks. With caching on, the +// run-specific tail that starts at VolatileSystemHeading goes in a second, +// unmarked block, so the cached prefix is the same for every run that shares +// the instructions and tools. The two blocks hold exactly the original text. +func systemBlocks(system string, cache bool) []apitypes.InputContentBlock { + if !cache { + return []apitypes.InputContentBlock{{Kind: "text", Text: system}} + } + stable, volatile := system, "" + if i := strings.LastIndex(system, "\n"+apitypes.VolatileSystemHeading+"\n"); i >= 0 { + stable, volatile = system[:i+1], system[i+1:] + } + // The API rejects whitespace-only text blocks. + if strings.TrimSpace(stable) == "" || strings.TrimSpace(volatile) == "" { + return []apitypes.InputContentBlock{{Kind: "text", Text: system, CacheControl: apitypes.EphemeralCache()}} + } + return []apitypes.InputContentBlock{ + {Kind: "text", Text: stable, CacheControl: apitypes.EphemeralCache()}, + {Kind: "text", Text: volatile}, + } +} + // withTrailingBreakpoint returns messages with a cache marker on the final // block of the final message, copying only what it changes. Any markers that // somehow reached the stored session are stripped so the count stays at one. diff --git a/internal/apiclient/anthropic_cache_test.go b/internal/apiclient/anthropic_cache_test.go index db89023..3b71df2 100644 --- a/internal/apiclient/anthropic_cache_test.go +++ b/internal/apiclient/anthropic_cache_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "net/http" + "strings" "testing" "github.com/AlleyBo55/gocode/internal/apitypes" @@ -221,3 +222,30 @@ func containsKey(v any, key string) bool { } return false } + +func TestAnthropicSystemCachesOnlyTheStablePrefix(t *testing.T) { + stable := "You are gocode.\n\n# Tools\n- read_file\n" + volatile := apitypes.VolatileSystemHeading + "\n\n- Working directory: /tmp/run-7\n- Current date: November 2, 2026\n" + wire := anthropicWireRequest(apitypes.MessageRequest{Model: "m", System: stable + "\n" + volatile, Messages: conversation()}, true) + + if len(wire.System) != 2 { + t.Fatalf("system = %+v, want a cached block and an uncached one", wire.System) + } + if wire.System[0].CacheControl == nil || wire.System[1].CacheControl != nil { + t.Errorf("only the first block may carry the marker: %+v", wire.System) + } + if !strings.HasPrefix(wire.System[1].Text, apitypes.VolatileSystemHeading) || strings.Contains(wire.System[0].Text, "/tmp/run-7") { + t.Errorf("split in the wrong place: %+v", wire.System) + } + if got := wire.System[0].Text + wire.System[1].Text; got != stable+"\n"+volatile { + t.Errorf("split changed the prompt text:\n%q", got) + } + + // Caching off, or nothing before the heading: one block, as before. + if plain := anthropicWireRequest(apitypes.MessageRequest{Model: "m", System: stable + "\n" + volatile}, false); len(plain.System) != 1 { + t.Errorf("caching off: system = %+v", plain.System) + } + if only := anthropicWireRequest(apitypes.MessageRequest{Model: "m", System: "\n" + volatile}, true); len(only.System) != 1 || only.System[0].CacheControl == nil { + t.Errorf("heading with no stable text before it: system = %+v", only.System) + } +} diff --git a/internal/apitypes/types.go b/internal/apitypes/types.go index 2d0468a..a793ed8 100644 --- a/internal/apitypes/types.go +++ b/internal/apitypes/types.go @@ -126,6 +126,13 @@ type CacheControl struct { // EphemeralCache is the cache marker to attach to a block. func EphemeralCache() *CacheControl { return &CacheControl{Type: "ephemeral"} } +// VolatileSystemHeading opens the last section of the system prompt: what +// changes from run to run (working directory, date, git state). Anthropic's +// provider caches only the text before it, so a run in another directory or +// on another day still reads the instructions and tool schemas from cache. +// Other providers send the prompt unchanged. +const VolatileSystemHeading = "# Environment" + // ToolDef is an LLM tool definition. type ToolDef struct { Name string `json:"name"` diff --git a/internal/repl/no_overthink.md b/internal/repl/no_overthink.md new file mode 100644 index 0000000..6e6f316 --- /dev/null +++ b/internal/repl/no_overthink.md @@ -0,0 +1,17 @@ +# No overthinking + +This rule comes before other instructions. When they conflict, follow this one, except on safety (destructive actions, data loss, security, secrets, money), where the stricter instruction wins. + +Think only when it clearly improves the result: multi-step work, debugging, risky changes. Otherwise answer directly without deliberating. + +- Take the first approach that meets the requirement. Weigh alternatives only when asked or when the choice is hard to undo (data loss, public interfaces, security, money). +- Treat earlier answers and decisions as settled unless the user reopens them or new evidence shows they are wrong. +- If a low-risk request is unclear, pick the likely reading, state it in one line, and proceed. Ask only when blocked or before destructive actions. +- Know what you're looking for before searching or reading, and stop when you find it. Don't re-read files already in context. +- Batch independent tool calls in parallel. +- Do everything asked and nothing more. Mention unrelated issues in one line instead of fixing them. +- Edit the lines that need to change; don't rewrite whole files. +- Verify once: run the checks that prove the change, and don't re-run them on unchanged code. +- If an approach fails twice, switch approaches or report the blocker. +- Don't ask permission for steps the request already covers. +- Lead with the answer or result. No preamble, no restating the request, no closing recap. Match length to the question. Give one recommendation, not a menu. diff --git a/internal/repl/repl.go b/internal/repl/repl.go index 8bdb4ab..647d38e 100644 --- a/internal/repl/repl.go +++ b/internal/repl/repl.go @@ -3,6 +3,7 @@ package repl import ( "bufio" "context" + _ "embed" "encoding/json" "fmt" "io" @@ -1185,8 +1186,17 @@ func RunOneShot(ctx context.Context, rt *agent.ConversationRuntime, prompt strin // SkipProjectConfig disables loading GOCODE.md/CLAUDE.md into the system prompt. var SkipProjectConfig bool -// BuildSystemPrompt constructs the system prompt for the agent. -// Modeled after Claude Code's agentic behavior — proactive, autonomous, thorough. +// noOverthinkRules is rules/no-overthink.md from +// github.com/AlleyBo55/no-overthink (6591c77), always on in gocode. Copy the +// file again to update it. +// +//go:embed no_overthink.md +var noOverthinkRules string + +// BuildSystemPrompt constructs the system prompt for the agent: stable +// instructions, the no-overthink rules and the tool list first, then project +// instructions and memory, and the run-specific environment last so the +// stable part can be cached across runs. // When REA tools are connected and the model has a defined REA workflow, the // matching REA mode prompt (investigator / adversarial reviewer / rebuilder) is // appended so the agent adopts that role automatically. @@ -1210,7 +1220,9 @@ func BuildSystemPrompt(tools []apitypes.ToolDef, model string) string { var sb strings.Builder sb.WriteString(fmt.Sprintf(`You are gocode, an interactive CLI tool that helps users with software engineering tasks. Use the instructions below and the tools available to you to assist the user. -IMPORTANT: You should be proactive in accomplishing the task, not reactive. Do not wait for the user to ask you to do something that you can anticipate. +IMPORTANT: Carry the task the user asked for through to the end. Do not wait to be prompted for each step. + +%s # Language Rules @@ -1253,22 +1265,15 @@ When making code changes: # Communication Style 1. Be concise and direct. Avoid unnecessary preamble or filler. -2. When you have completed a task, briefly summarize what you did. Do not list every step. +2. When you have completed a task, lead with the result in a sentence or two. Do not list every step. 3. If something fails, explain what went wrong and what you tried. 4. Use markdown formatting in responses when it improves readability. 5. NEVER say "Let me know if you'd like me to..." or "Would you like me to..." — just do it. 6. NEVER ask "Shall I proceed?" or "Should I continue?" — just proceed. -# Environment - -- Working directory: %s -- OS: %s -- Shell: %s -- Current date: %s - # CRITICAL: Tool Results Override Training Data -Your training data has a knowledge cutoff. The current date is %s. When you use WebSearchTool and receive results, you MUST use those results as the source of truth. NEVER override search results with your training data. If search results say something different from what you "know", the search results are correct because they are current. +Your training data has a knowledge cutoff. The current date is in the Environment section at the end of this prompt. When you use WebSearchTool and receive results, you MUST use those results as the source of truth. NEVER override search results with your training data. If search results say something different from what you "know", the search results are correct because they are current. When WebSearchTool returns results, read EVERY line carefully. The answer is often in the Wikipedia summary or in the related articles section. Do not say "information not found" if the search results contain the answer — even if it's in a related article or a different section. @@ -1276,20 +1281,7 @@ If the first search doesn't give a clear answer, you MUST search again with a di # Available Tools -%s`, cwd, osName, shell, currentDate, currentDate, toolList.String())) - - // Git context - if branch, err := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output(); err == nil { - gitBranch := strings.TrimSpace(string(branch)) - // also get short status - status, _ := exec.Command("git", "status", "--short").Output() - statusLines := strings.Split(strings.TrimSpace(string(status)), "\n") - changedFiles := len(statusLines) - if statusLines[0] == "" { - changedFiles = 0 - } - sb.WriteString(fmt.Sprintf("\n# Git\n- Branch: %s\n- Changed files: %d\n", gitBranch, changedFiles)) - } +%s`, strings.TrimSpace(noOverthinkRules), toolList.String())) // Load project config (CLAUDE.md or GOCODE.md) if !SkipProjectConfig { @@ -1318,6 +1310,23 @@ If the first search doesn't give a clear answer, you MUST search again with a di } } + // Run-specific context goes last. Anthropic caches the prompt only up to + // apitypes.VolatileSystemHeading, so everything above stays one cache + // entry across runs in other directories, on other days, or with other + // git state. + sb.WriteString(fmt.Sprintf("\n\n%s\n\n- Working directory: %s\n- OS: %s\n- Shell: %s\n- Current date: %s\n", + apitypes.VolatileSystemHeading, cwd, osName, shell, currentDate)) + if branch, err := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output(); err == nil { + gitBranch := strings.TrimSpace(string(branch)) + status, _ := exec.Command("git", "status", "--short").Output() + statusLines := strings.Split(strings.TrimSpace(string(status)), "\n") + changedFiles := len(statusLines) + if statusLines[0] == "" { + changedFiles = 0 + } + sb.WriteString(fmt.Sprintf("\n# Git\n- Branch: %s\n- Changed files: %d\n", gitBranch, changedFiles)) + } + return sb.String() }