From 4708b4e160fb0432d2fa318a7b0e041441b07a39 Mon Sep 17 00:00:00 2001
From: Colum Ferry <cferry09@gmail.com>
Date: Tue, 3 Nov 2026 15:51:08 +0000
Subject: [PATCH 1/7] feat(cli): run the cli suite on its own daily and weekly
 schedule

Add cli-eval-schedule.yml, owned by @supabase/cli, which dispatches
eval-refresh for the cli suite: codex-gpt-6-luna on every channel and
Docker state daily, and a five-model matrix on stable and next weekly.
Both runs export with merge, so their rows share cli-eval-results.json
and its gh-pages history.

eval-refresh gains an auto_merge dispatch input so these runs merge their
results PR unattended, and the shared nightly stops running the cli suite.

Closes CLI-2714
Closes CLI-2715
---
 .github/CODEOWNERS                            |  1 +
 .github/workflows/cli-eval-schedule.yml       | 76 +++++++++++++++++++
 .github/workflows/eval-refresh.yml            | 17 +++--
 CONTRIBUTING.md                               |  9 ++-
 ...laude-code-opus-5.5-cli-next.experiment.ts | 11 +++
 ...ude-code-opus-5.5-cli-stable.experiment.ts | 11 +++
 ...ude-code-sonnet-5.5-cli-next.experiment.ts | 11 +++
 ...e-code-sonnet-5.5-cli-stable.experiment.ts | 11 +++
 .../codex-gpt-6.1-sol-cli-next.experiment.ts  | 11 +++
 ...codex-gpt-6.1-sol-cli-stable.experiment.ts | 11 +++
 .../opencode-kimi-k3-cli-next.experiment.ts   | 11 +++
 .../opencode-kimi-k3-cli-stable.experiment.ts | 11 +++
 12 files changed, 184 insertions(+), 7 deletions(-)
 create mode 100644 .github/workflows/cli-eval-schedule.yml
 create mode 100644 experiments/cli/claude-code-opus-5.5-cli-next.experiment.ts
 create mode 100644 experiments/cli/claude-code-opus-5.5-cli-stable.experiment.ts
 create mode 100644 experiments/cli/claude-code-sonnet-5.5-cli-next.experiment.ts
 create mode 100644 experiments/cli/claude-code-sonnet-5.5-cli-stable.experiment.ts
 create mode 100644 experiments/cli/codex-gpt-6.1-sol-cli-next.experiment.ts
 create mode 100644 experiments/cli/codex-gpt-6.1-sol-cli-stable.experiment.ts
 create mode 100644 experiments/cli/opencode-kimi-k3-cli-next.experiment.ts
 create mode 100644 experiments/cli/opencode-kimi-k3-cli-stable.experiment.ts

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index f63d16b1..aa9e5a2a 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -8,4 +8,5 @@
 
 /evals/cli/ @supabase/cli
 /experiments/cli/ @supabase/cli
+/.github/workflows/cli-eval-schedule.yml @supabase/cli
 /apps/web/src/data/cli-eval-results.json @supabase/cli
diff --git a/.github/workflows/cli-eval-schedule.yml b/.github/workflows/cli-eval-schedule.yml
new file mode 100644
index 00000000..3af983f2
--- /dev/null
+++ b/.github/workflows/cli-eval-schedule.yml
@@ -0,0 +1,76 @@
+# The CLI team owns this cadence and reuses eval-refresh's run and publish pipeline.
+# merge=true keeps the daily and weekly rows together in cli-eval-results.json.
+name: CLI eval schedule
+
+on:
+  schedule:
+    - cron: '20 4 * * *'
+    # CADENCE below picks the weekly list by matching this string.
+    - cron: '20 1 * * 1'
+  workflow_dispatch:
+    inputs:
+      cadence:
+        description: "Which experiment list to run"
+        type: choice
+        options:
+          - daily
+          - weekly
+        default: daily
+
+permissions:
+  actions: write
+
+env:
+  DAILY_EXPERIMENTS: >-
+    codex-gpt-6-luna-cli-pinned,
+    codex-gpt-6-luna-cli-stable,
+    codex-gpt-6-luna-cli-beta,
+    codex-gpt-6-luna-cli-next,
+    codex-gpt-6-luna-cli-nodaemon,
+    codex-gpt-6-luna-cli-absent
+  WEEKLY_EXPERIMENTS: >-
+    claude-code-sonnet-5.5-cli-stable,
+    claude-code-sonnet-5.5-cli-next,
+    claude-code-opus-5.5-cli-stable,
+    claude-code-opus-5.5-cli-next,
+    codex-gpt-6.1-sol-cli-stable,
+    codex-gpt-6.1-sol-cli-next,
+    codex-gpt-6-luna-cli-stable,
+    codex-gpt-6-luna-cli-next,
+    opencode-kimi-k3-cli-stable,
+    opencode-kimi-k3-cli-next
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-26.04
+    timeout-minutes: 5
+    steps:
+      - name: Dispatch eval-refresh
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          CADENCE: ${{ github.event_name == 'workflow_dispatch' && inputs.cadence || github.event.schedule == '20 1 * * 1' && 'weekly' || 'daily' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          case "$CADENCE" in
+            daily) experiments="$DAILY_EXPERIMENTS" ;;
+            weekly) experiments="$WEEKLY_EXPERIMENTS" ;;
+            *)
+              echo "::error::Unknown cadence: $CADENCE"
+              exit 1
+              ;;
+          esac
+
+          gh workflow run eval-refresh.yml \
+            --ref "$GITHUB_REF_NAME" \
+            -f suite=cli \
+            -f experiments="$experiments" \
+            -f runs=3 \
+            -f timeout_sec=720 \
+            -f sandbox_concurrency=250 \
+            -f merge=true \
+            -f auto_merge=true
+
+          echo "Dispatched eval-refresh for the $CADENCE cadence: $experiments" >> "$GITHUB_STEP_SUMMARY"
diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index 24c16b7d..0ab1cb5c 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -37,6 +37,11 @@ on:
         type: boolean
         required: false
         default: false
+      auto_merge:
+        description: "Merge the results PR once every pair completes (for scheduled callers such as cli-eval-schedule.yml)"
+        type: boolean
+        required: false
+        default: false
       cli_stable_version:
         description: "Pin the cli suite's stable CLI version instead of resolving npm's latest dist-tag (blank to resolve)"
         required: false
@@ -118,7 +123,8 @@ jobs:
           elif [ "${{ github.event_name }}" = "schedule" ]; then
             experiments_override=""
             eval_id=""
-            suite="regression,cli"
+            # The cli suite runs on its own cadence in cli-eval-schedule.yml.
+            suite="regression"
             runs="3"
             timeout_sec="720"
             sandbox_concurrency="250"
@@ -579,17 +585,18 @@ jobs:
           title: "chore: refresh eval results"
           body: |
             Refreshes `apps/web/src/data/eval-results.json` from the latest automated eval run.
-          # Draft PRs can't be merged, and the scheduled path merges itself.
-          draft: ${{ github.event_name != 'schedule' }}
+          # Draft PRs can't be merged, and scheduled and auto_merge runs merge themselves.
+          draft: ${{ github.event_name != 'schedule' && !inputs.auto_merge }}
           delete-branch: true
 
-      - name: Merge scheduled results pull request
+      - name: Merge results pull request
         # --admin merges without the required review, which the bypass list
         # entitles the app to.
         # https://cli.github.com/manual/gh_pr_merge
         # Skips the merge when results are partial, so it stays open for review.
         if: >-
-          github.event_name == 'schedule' &&
+          (github.event_name == 'schedule' ||
+            (github.event_name == 'workflow_dispatch' && inputs.auto_merge)) &&
           steps.cpr.outputs.pull-request-number &&
           steps.audit.outputs.count == '0'
         env:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0add5ba7..7196836b 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -102,11 +102,11 @@ Common workflows:
 
 ## CLI evals
 
-The CLI team owns `evals/cli/` and its results. CLI evals run on `codex-gpt-6-luna-cli-{pinned,stable,beta,next,nodaemon,absent}` under `experiments/cli/`: `pinned` runs the repo's pinned CLI version, `stable`/`beta`/`next` install the latest stable, beta or next CLI, and `nodaemon`/`absent` additionally force Docker-less sandboxes — comparing the same scenario across CLI environments.
+The CLI team owns `evals/cli/` and its results. CLI evals run on `codex-gpt-6-luna-cli-{pinned,stable,beta,next,nodaemon,absent}` under `experiments/cli/`: `pinned` runs the repo's pinned CLI version, `stable`/`beta`/`next` install the latest stable, beta or next CLI, and `nodaemon`/`absent` additionally force Docker-less sandboxes — comparing the same scenario across CLI environments. The `<model>-cli-{stable,next}` arms run the same scenario on the stable and next CLI across `claude-code-sonnet-5.5`, `claude-code-opus-5.5`, `codex-gpt-6.1-sol` and `opencode-kimi-k3`, comparing models rather than environments.
 
 Which evals each arm picks up:
 
-- **pinned, stable, beta, next** run every `interface: cli` eval that isn't `hostedProject: true`.
+- **pinned, stable, beta, next** run every `interface: cli` eval that isn't `hostedProject: true`. The `<model>-cli-{stable,next}` matrix arms pick up the same evals as `stable` and `next`.
 - **nodaemon, absent** additionally only run evals that also set `needsDocker: false` and `projectRunning: false` — the harness cannot pre-start a stack, or link a hosted project, without Docker.
 
 Common workflows:
@@ -114,4 +114,9 @@ Common workflows:
 - **Add or change a CLI eval.** Add the scenario under `evals/cli/<id>/` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview.
 - **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version`/`cli_next_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run.
 - **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories.
+- **Scheduled runs.** The CLI team's schedule dispatches [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) with `suite: cli` and `merge`, so daily and weekly rows share `cli-eval-results.json`. Each run auto-merges its results PR when every pair completes.
+  - Daily, 04:20 UTC: the six `codex-gpt-6-luna-cli-*` arms.
+  - Weekly, Monday 01:20 UTC: the ten-arm matrix (five models on stable and next).
+  - Edit the cadence and experiment lists in `.github/workflows/cli-eval-schedule.yml`.
+  - The shared nightly no longer runs the cli suite.
 - **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm test`.
diff --git a/experiments/cli/claude-code-opus-5.5-cli-next.experiment.ts b/experiments/cli/claude-code-opus-5.5-cli-next.experiment.ts
new file mode 100644
index 00000000..62d9b209
--- /dev/null
+++ b/experiments/cli/claude-code-opus-5.5-cli-next.experiment.ts
@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { claudeCodeOpus55 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...claudeCodeOpus55,
+  localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});
diff --git a/experiments/cli/claude-code-opus-5.5-cli-stable.experiment.ts b/experiments/cli/claude-code-opus-5.5-cli-stable.experiment.ts
new file mode 100644
index 00000000..c1f15941
--- /dev/null
+++ b/experiments/cli/claude-code-opus-5.5-cli-stable.experiment.ts
@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { claudeCodeOpus55 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...claudeCodeOpus55,
+  localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});
diff --git a/experiments/cli/claude-code-sonnet-5.5-cli-next.experiment.ts b/experiments/cli/claude-code-sonnet-5.5-cli-next.experiment.ts
new file mode 100644
index 00000000..68204be6
--- /dev/null
+++ b/experiments/cli/claude-code-sonnet-5.5-cli-next.experiment.ts
@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { claudeCodeSonnet55 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...claudeCodeSonnet55,
+  localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});
diff --git a/experiments/cli/claude-code-sonnet-5.5-cli-stable.experiment.ts b/experiments/cli/claude-code-sonnet-5.5-cli-stable.experiment.ts
new file mode 100644
index 00000000..fdd0b12d
--- /dev/null
+++ b/experiments/cli/claude-code-sonnet-5.5-cli-stable.experiment.ts
@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { claudeCodeSonnet55 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...claudeCodeSonnet55,
+  localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});
diff --git a/experiments/cli/codex-gpt-6.1-sol-cli-next.experiment.ts b/experiments/cli/codex-gpt-6.1-sol-cli-next.experiment.ts
new file mode 100644
index 00000000..92852295
--- /dev/null
+++ b/experiments/cli/codex-gpt-6.1-sol-cli-next.experiment.ts
@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { codexGpt61Sol } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...codexGpt61Sol,
+  localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});
diff --git a/experiments/cli/codex-gpt-6.1-sol-cli-stable.experiment.ts b/experiments/cli/codex-gpt-6.1-sol-cli-stable.experiment.ts
new file mode 100644
index 00000000..654e2892
--- /dev/null
+++ b/experiments/cli/codex-gpt-6.1-sol-cli-stable.experiment.ts
@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { codexGpt61Sol } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...codexGpt61Sol,
+  localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});
diff --git a/experiments/cli/opencode-kimi-k3-cli-next.experiment.ts b/experiments/cli/opencode-kimi-k3-cli-next.experiment.ts
new file mode 100644
index 00000000..d5918e02
--- /dev/null
+++ b/experiments/cli/opencode-kimi-k3-cli-next.experiment.ts
@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { opencodeKimiK3 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...opencodeKimiK3,
+  localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});
diff --git a/experiments/cli/opencode-kimi-k3-cli-stable.experiment.ts b/experiments/cli/opencode-kimi-k3-cli-stable.experiment.ts
new file mode 100644
index 00000000..970fb49a
--- /dev/null
+++ b/experiments/cli/opencode-kimi-k3-cli-stable.experiment.ts
@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { opencodeKimiK3 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...opencodeKimiK3,
+  localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

From 300b35b379513c75a60c2a7b7734feb03f7b28b0 Mon Sep 17 00:00:00 2001
From: Colum Ferry <cferry09@gmail.com>
Date: Wed, 4 Nov 2026 14:07:34 +0000
Subject: [PATCH 2/7] feat(cli): record the daily or weekly cadence on
 braintrust experiments

---
 .github/workflows/cli-eval-schedule.yml       |  1 +
 .github/workflows/eval-refresh.yml            |  6 ++++++
 CONTRIBUTING.md                               |  2 +-
 .../scripts/upload-braintrust.test.ts         | 20 +++++++++++++++++++
 apps/framework/scripts/upload-braintrust.ts   |  6 ++++++
 5 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/cli-eval-schedule.yml b/.github/workflows/cli-eval-schedule.yml
index 3af983f2..2dc1c170 100644
--- a/.github/workflows/cli-eval-schedule.yml
+++ b/.github/workflows/cli-eval-schedule.yml
@@ -66,6 +66,7 @@ jobs:
           gh workflow run eval-refresh.yml \
             --ref "$GITHUB_REF_NAME" \
             -f suite=cli \
+            -f cadence="$CADENCE" \
             -f experiments="$experiments" \
             -f runs=3 \
             -f timeout_sec=720 \
diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index 0ab1cb5c..ae51f930 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -54,6 +54,11 @@ on:
         description: "Pin the cli suite's next CLI version instead of resolving npm's next dist-tag (blank to resolve)"
         required: false
         default: ""
+      cadence:
+        description: "Schedule that dispatched this run (daily or weekly); recorded on each Braintrust experiment"
+        type: string
+        required: false
+        default: ""
   schedule:
     - cron: '15 6 * * *'
   pull_request:
@@ -527,6 +532,7 @@ jobs:
         env:
           BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
           BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}
+          EVAL_CADENCE: ${{ inputs.cadence }}
         run: pnpm upload-braintrust --runs "${{ needs.prepare.outputs.runs }}"
 
       - name: Upload exported results
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7196836b..78eb9b50 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -114,7 +114,7 @@ Common workflows:
 - **Add or change a CLI eval.** Add the scenario under `evals/cli/<id>/` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview.
 - **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version`/`cli_next_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run.
 - **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories.
-- **Scheduled runs.** The CLI team's schedule dispatches [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) with `suite: cli` and `merge`, so daily and weekly rows share `cli-eval-results.json`. Each run auto-merges its results PR when every pair completes.
+- **Scheduled runs.** The CLI team's schedule dispatches [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) with `suite: cli` and `merge`, so daily and weekly rows share `cli-eval-results.json`; each run's Braintrust experiments carry `cadence: daily|weekly`. Each run auto-merges its results PR when every pair completes.
   - Daily, 04:20 UTC: the six `codex-gpt-6-luna-cli-*` arms.
   - Weekly, Monday 01:20 UTC: the ten-arm matrix (five models on stable and next).
   - Edit the cadence and experiment lists in `.github/workflows/cli-eval-schedule.yml`.
diff --git a/apps/framework/scripts/upload-braintrust.test.ts b/apps/framework/scripts/upload-braintrust.test.ts
index 322c178d..7ee98b50 100644
--- a/apps/framework/scripts/upload-braintrust.test.ts
+++ b/apps/framework/scripts/upload-braintrust.test.ts
@@ -1,5 +1,6 @@
 import { afterEach, describe, expect, it, vi } from 'vitest';
 import {
+  cadence,
   ciRunUrl,
   experimentName,
   judgeMetrics,
@@ -139,6 +140,25 @@ describe('ciRunUrl', () => {
   });
 });
 
+describe('cadence', () => {
+  afterEach(() => vi.unstubAllEnvs());
+
+  it('reports the dispatching schedule when set', () => {
+    vi.stubEnv('EVAL_CADENCE', ' weekly ');
+    expect(cadence()).toEqual({ cadence: 'weekly' });
+  });
+
+  it('is omitted when unset, empty, or whitespace', () => {
+    vi.stubEnv('EVAL_CADENCE', '');
+    expect(cadence()).toEqual({});
+    vi.stubEnv('EVAL_CADENCE', '  ');
+    expect(cadence()).toEqual({});
+    vi.unstubAllEnvs();
+    delete process.env.EVAL_CADENCE;
+    expect(cadence()).toEqual({});
+  });
+});
+
 describe('runViewUrl', () => {
   it('filters the experiments list by run id', () => {
     const url = runViewUrl(
diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts
index 6f4e617d..1c25c680 100644
--- a/apps/framework/scripts/upload-braintrust.ts
+++ b/apps/framework/scripts/upload-braintrust.ts
@@ -317,6 +317,11 @@ export function ciRunUrl() {
     : {};
 }
 
+export function cadence() {
+  const value = process.env.EVAL_CADENCE?.trim();
+  return value ? { cadence: value } : {};
+}
+
 async function collectRows(
   experimentMetadata: Map<string, ExperimentMetadata>
 ): Promise<Map<string, ExperimentGroup>> {
@@ -807,6 +812,7 @@ async function main() {
         ...(info?.branch ? { branch: info.branch } : {}),
         ...prUrl(),
         ...ciRunUrl(),
+        ...cadence(),
       },
     });
 

From 9a291aaf115cae50044e707d81c75f551c2a8aeb Mon Sep 17 00:00:00 2001
From: Colum Ferry <cferry09@gmail.com>
Date: Wed, 4 Nov 2026 14:20:14 +0000
Subject: [PATCH 3/7] feat(cli): publish scheduled cli runs to braintrust only

---
 .github/workflows/cli-eval-schedule.yml |  5 ++---
 .github/workflows/eval-refresh.yml      | 21 ++++++++++-----------
 CONTRIBUTING.md                         |  2 +-
 3 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/cli-eval-schedule.yml b/.github/workflows/cli-eval-schedule.yml
index 2dc1c170..3aab57b8 100644
--- a/.github/workflows/cli-eval-schedule.yml
+++ b/.github/workflows/cli-eval-schedule.yml
@@ -1,5 +1,5 @@
 # The CLI team owns this cadence and reuses eval-refresh's run and publish pipeline.
-# merge=true keeps the daily and weekly rows together in cli-eval-results.json.
+# Results go to Braintrust only; no cli-eval-results.json PR is opened.
 name: CLI eval schedule
 
 on:
@@ -71,7 +71,6 @@ jobs:
             -f runs=3 \
             -f timeout_sec=720 \
             -f sandbox_concurrency=250 \
-            -f merge=true \
-            -f auto_merge=true
+            -f results_pr=false
 
           echo "Dispatched eval-refresh for the $CADENCE cadence: $experiments" >> "$GITHUB_STEP_SUMMARY"
diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index ae51f930..737d156f 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -37,11 +37,11 @@ on:
         type: boolean
         required: false
         default: false
-      auto_merge:
-        description: "Merge the results PR once every pair completes (for scheduled callers such as cli-eval-schedule.yml)"
+      results_pr:
+        description: "Open a PR with the exported results JSON (off for Braintrust-only callers such as cli-eval-schedule.yml)"
         type: boolean
         required: false
-        default: false
+        default: true
       cli_stable_version:
         description: "Pin the cli suite's stable CLI version instead of resolving npm's latest dist-tag (blank to resolve)"
         required: false
@@ -527,8 +527,8 @@ jobs:
           echo "PR_URLS=$urls" >> "$GITHUB_ENV"
 
       - name: Upload results to Braintrust
-        # A Braintrust outage shouldn't block publishing results.
-        continue-on-error: true
+        # A Braintrust outage shouldn't block publishing results, but with no results PR it is the only record.
+        continue-on-error: ${{ github.event_name != 'workflow_dispatch' || inputs.results_pr }}
         env:
           BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
           BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}
@@ -576,7 +576,7 @@ jobs:
       - name: Create results pull request
         if: >-
           github.event_name == 'schedule' ||
-          (github.event_name == 'workflow_dispatch' && !inputs.commit_to_branch)
+          (github.event_name == 'workflow_dispatch' && !inputs.commit_to_branch && inputs.results_pr)
         id: cpr
         uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
         with:
@@ -591,18 +591,17 @@ jobs:
           title: "chore: refresh eval results"
           body: |
             Refreshes `apps/web/src/data/eval-results.json` from the latest automated eval run.
-          # Draft PRs can't be merged, and scheduled and auto_merge runs merge themselves.
-          draft: ${{ github.event_name != 'schedule' && !inputs.auto_merge }}
+          # Draft PRs can't be merged, and the scheduled path merges itself.
+          draft: ${{ github.event_name != 'schedule' }}
           delete-branch: true
 
-      - name: Merge results pull request
+      - name: Merge scheduled results pull request
         # --admin merges without the required review, which the bypass list
         # entitles the app to.
         # https://cli.github.com/manual/gh_pr_merge
         # Skips the merge when results are partial, so it stays open for review.
         if: >-
-          (github.event_name == 'schedule' ||
-            (github.event_name == 'workflow_dispatch' && inputs.auto_merge)) &&
+          github.event_name == 'schedule' &&
           steps.cpr.outputs.pull-request-number &&
           steps.audit.outputs.count == '0'
         env:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 78eb9b50..7c710e11 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -114,7 +114,7 @@ Common workflows:
 - **Add or change a CLI eval.** Add the scenario under `evals/cli/<id>/` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview.
 - **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version`/`cli_next_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run.
 - **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories.
-- **Scheduled runs.** The CLI team's schedule dispatches [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) with `suite: cli` and `merge`, so daily and weekly rows share `cli-eval-results.json`; each run's Braintrust experiments carry `cadence: daily|weekly`. Each run auto-merges its results PR when every pair completes.
+- **Scheduled runs.** The CLI team's schedule dispatches [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) with `suite: cli` and `results_pr: false`, so its results go to the Evals project in Braintrust only, tagged `cadence: daily|weekly`, and don't update `cli-eval-results.json`.
   - Daily, 04:20 UTC: the six `codex-gpt-6-luna-cli-*` arms.
   - Weekly, Monday 01:20 UTC: the ten-arm matrix (five models on stable and next).
   - Edit the cadence and experiment lists in `.github/workflows/cli-eval-schedule.yml`.

From df901937c4786e7b03af1b73d44b5c4abb661958 Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 5 Nov 2026 18:14:10 -0500
Subject: [PATCH 4/7] refactor: reuse eval refresh workflow for schedules

---
 .github/workflows/cli-eval-schedule.yml       |  95 ++++++-------
 .github/workflows/eval-refresh.yml            | 130 +++++++++++-------
 .../workflows/regression-eval-schedule.yml    |  18 +++
 CONTRIBUTING.md                               |   4 +-
 .../scripts/upload-braintrust.test.ts         |  19 +--
 apps/framework/scripts/upload-braintrust.ts   |   6 +-
 6 files changed, 145 insertions(+), 127 deletions(-)
 create mode 100644 .github/workflows/regression-eval-schedule.yml

diff --git a/.github/workflows/cli-eval-schedule.yml b/.github/workflows/cli-eval-schedule.yml
index 3aab57b8..64cc9ebe 100644
--- a/.github/workflows/cli-eval-schedule.yml
+++ b/.github/workflows/cli-eval-schedule.yml
@@ -1,11 +1,8 @@
-# The CLI team owns this cadence and reuses eval-refresh's run and publish pipeline.
-# Results go to Braintrust only; no cli-eval-results.json PR is opened.
 name: CLI eval schedule
 
 on:
   schedule:
     - cron: '20 4 * * *'
-    # CADENCE below picks the weekly list by matching this string.
     - cron: '20 1 * * 1'
   workflow_dispatch:
     inputs:
@@ -18,59 +15,45 @@ on:
         default: daily
 
 permissions:
-  actions: write
-
-env:
-  DAILY_EXPERIMENTS: >-
-    codex-gpt-6-luna-cli-pinned,
-    codex-gpt-6-luna-cli-stable,
-    codex-gpt-6-luna-cli-beta,
-    codex-gpt-6-luna-cli-next,
-    codex-gpt-6-luna-cli-nodaemon,
-    codex-gpt-6-luna-cli-absent
-  WEEKLY_EXPERIMENTS: >-
-    claude-code-sonnet-5.5-cli-stable,
-    claude-code-sonnet-5.5-cli-next,
-    claude-code-opus-5.5-cli-stable,
-    claude-code-opus-5.5-cli-next,
-    codex-gpt-6.1-sol-cli-stable,
-    codex-gpt-6.1-sol-cli-next,
-    codex-gpt-6-luna-cli-stable,
-    codex-gpt-6-luna-cli-next,
-    opencode-kimi-k3-cli-stable,
-    opencode-kimi-k3-cli-next
+  contents: write
+  pull-requests: write
+  actions: read
 
 jobs:
-  dispatch:
-    runs-on: ubuntu-26.04
-    timeout-minutes: 5
-    steps:
-      - name: Dispatch eval-refresh
-        env:
-          GH_TOKEN: ${{ github.token }}
-          GH_REPO: ${{ github.repository }}
-          CADENCE: ${{ github.event_name == 'workflow_dispatch' && inputs.cadence || github.event.schedule == '20 1 * * 1' && 'weekly' || 'daily' }}
-        shell: bash
-        run: |
-          set -euo pipefail
-
-          case "$CADENCE" in
-            daily) experiments="$DAILY_EXPERIMENTS" ;;
-            weekly) experiments="$WEEKLY_EXPERIMENTS" ;;
-            *)
-              echo "::error::Unknown cadence: $CADENCE"
-              exit 1
-              ;;
-          esac
-
-          gh workflow run eval-refresh.yml \
-            --ref "$GITHUB_REF_NAME" \
-            -f suite=cli \
-            -f cadence="$CADENCE" \
-            -f experiments="$experiments" \
-            -f runs=3 \
-            -f timeout_sec=720 \
-            -f sandbox_concurrency=250 \
-            -f results_pr=false
+  daily:
+    if: github.event.schedule == '20 4 * * *' || (github.event_name == 'workflow_dispatch' && inputs.cadence == 'daily')
+    uses: ./.github/workflows/eval-refresh.yml
+    secrets: inherit
+    with:
+      suite: cli
+      merge: false
+      cadence: daily
+      commit_results: none
+      experiments: >-
+        codex-gpt-6-luna-cli-pinned,
+        codex-gpt-6-luna-cli-stable,
+        codex-gpt-6-luna-cli-beta,
+        codex-gpt-6-luna-cli-next,
+        codex-gpt-6-luna-cli-nodaemon,
+        codex-gpt-6-luna-cli-absent
 
-          echo "Dispatched eval-refresh for the $CADENCE cadence: $experiments" >> "$GITHUB_STEP_SUMMARY"
+  weekly:
+    if: github.event.schedule == '20 1 * * 1' || (github.event_name == 'workflow_dispatch' && inputs.cadence == 'weekly')
+    uses: ./.github/workflows/eval-refresh.yml
+    secrets: inherit
+    with:
+      suite: cli
+      merge: false
+      cadence: weekly
+      commit_results: none
+      experiments: >-
+        claude-code-sonnet-5.5-cli-stable,
+        claude-code-sonnet-5.5-cli-next,
+        claude-code-opus-5.5-cli-stable,
+        claude-code-opus-5.5-cli-next,
+        codex-gpt-6.1-sol-cli-stable,
+        codex-gpt-6.1-sol-cli-next,
+        codex-gpt-6-luna-cli-stable,
+        codex-gpt-6-luna-cli-next,
+        opencode-kimi-k3-cli-stable,
+        opencode-kimi-k3-cli-next
diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index 737d156f..c3ee939a 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -3,64 +3,87 @@ name: Refresh eval results
 on:
   workflow_dispatch:
     inputs:
-      experiments:
+      experiments: &experiments
         description: "Comma-separated experiment names to run within each selected eval suite"
+        type: string
         required: false
         default: ""
-      eval:
+      eval: &eval
         description: "Comma-separated eval ids to run (overrides suite)"
+        type: string
         required: false
         default: ""
-      suite:
+      suite: &suite
         description: "Comma-separated suites to run (evals/<suite> with experiments/<suite>)"
+        type: string
         required: false
         default: ""
-      runs:
+      runs: &runs
         description: "Independent scored runs per experiment/eval pair"
-        required: true
+        type: string
         default: "3"
-      timeout_sec:
+      timeout_sec: &timeout_sec
         description: "Timeout per run in seconds"
-        required: true
+        type: string
         default: "720"
-      sandbox_concurrency:
+      sandbox_concurrency: &sandbox_concurrency
         description: "Maximum Vercel Sandboxes running at once"
-        required: true
+        type: string
         default: "250"
-      merge:
+      merge: &merge
         description: "Merge into existing results instead of overwriting (graft new experiment/eval pairs)"
         type: boolean
         required: false
         default: true
-      commit_to_branch:
-        description: "Commit exported results to the dispatched branch instead of opening a PR"
-        type: boolean
-        required: false
-        default: false
-      results_pr:
-        description: "Open a PR with the exported results JSON (off for Braintrust-only callers such as cli-eval-schedule.yml)"
-        type: boolean
-        required: false
-        default: true
-      cli_stable_version:
+      commit_results:
+        description: "Where to commit exported results JSON"
+        type: choice
+        options:
+          - results-pr
+          - current-branch
+          - none
+        default: results-pr
+      cli_stable_version: &cli_stable_version
         description: "Pin the cli suite's stable CLI version instead of resolving npm's latest dist-tag (blank to resolve)"
+        type: string
         required: false
         default: ""
-      cli_beta_version:
+      cli_beta_version: &cli_beta_version
         description: "Pin the cli suite's beta CLI version instead of resolving npm's beta dist-tag (blank to resolve)"
+        type: string
         required: false
         default: ""
-      cli_next_version:
+      cli_next_version: &cli_next_version
         description: "Pin the cli suite's next CLI version instead of resolving npm's next dist-tag (blank to resolve)"
+        type: string
         required: false
         default: ""
       cadence:
-        description: "Schedule that dispatched this run (daily or weekly); recorded on each Braintrust experiment"
+        description: "Schedule recorded on each Braintrust experiment"
+        type: choice
+        options:
+          - none
+          - daily
+          - weekly
+        default: none
+  workflow_call:
+    inputs:
+      experiments: *experiments
+      eval: *eval
+      suite: *suite
+      runs: *runs
+      timeout_sec: *timeout_sec
+      sandbox_concurrency: *sandbox_concurrency
+      merge: *merge
+      cli_stable_version: *cli_stable_version
+      cli_beta_version: *cli_beta_version
+      cli_next_version: *cli_next_version
+      cadence:
         type: string
-        required: false
         default: ""
-  schedule:
-    - cron: '15 6 * * *'
+      commit_results:
+        type: string
+        default: results-pr
   pull_request:
     # Run whenever a PR carrying the run-evals label is opened, pushed to, or
     # receives the run-evals label. The job-level `if` gates on those cases.
@@ -73,9 +96,9 @@ permissions:
 
 concurrency:
   # The results app's pushes get a unique group so they can't cancel the run that pushed them.
-  group: eval-refresh-${{ github.event.sender.id == 291952723 && github.run_id || github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}
+  group: eval-refresh-${{ github.workflow }}-${{ github.event.sender.id == 291952723 && github.run_id || github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}
   # Supersede an in-flight run when a new commit is pushed to the same PR, but
-  # never cancel a workflow_dispatch run (those open the refresh PR).
+  # never cancel a workflow_dispatch run.
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
@@ -107,10 +130,25 @@ jobs:
       - name: Prepare inputs
         id: inputs
         shell: bash
+        env:
+          EVAL_CADENCE: ${{ inputs.cadence != 'none' && inputs.cadence || '' }}
+          COMMIT_RESULTS: ${{ inputs.commit_results }}
         run: |
           set -euo pipefail
 
-          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+          if [ "${{ github.event_name }}" != "pull_request" ]; then
+            case "$COMMIT_RESULTS" in
+              results-pr|current-branch|none) ;;
+              *) echo "::error::Invalid commit_results: $COMMIT_RESULTS"; exit 1 ;;
+            esac
+          fi
+
+          case "$EVAL_CADENCE" in
+            ""|daily|weekly) ;;
+            *) echo "::error::Invalid cadence: $EVAL_CADENCE"; exit 1 ;;
+          esac
+
+          if [ "${{ github.event_name }}" != "pull_request" ]; then
             experiments_override="${{ inputs.experiments }}"
             eval_id="${{ inputs.eval }}"
             suite="${{ inputs.suite }}"
@@ -125,17 +163,6 @@ jobs:
               echo "::error::Set suite or eval to choose which evals to run."
               exit 1
             fi
-          elif [ "${{ github.event_name }}" = "schedule" ]; then
-            experiments_override=""
-            eval_id=""
-            # The cli suite runs on its own cadence in cli-eval-schedule.yml.
-            suite="regression"
-            runs="3"
-            timeout_sec="720"
-            sandbox_concurrency="250"
-            cli_stable_version=""
-            cli_beta_version=""
-            cli_next_version=""
           else
             experiments_override=""
             eval_id=""
@@ -528,12 +555,17 @@ jobs:
 
       - name: Upload results to Braintrust
         # A Braintrust outage shouldn't block publishing results, but with no results PR it is the only record.
-        continue-on-error: ${{ github.event_name != 'workflow_dispatch' || inputs.results_pr }}
+        continue-on-error: ${{ github.event_name == 'pull_request' || inputs.commit_results != 'none' }}
         env:
           BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
           BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}
-          EVAL_CADENCE: ${{ inputs.cadence }}
-        run: pnpm upload-braintrust --runs "${{ needs.prepare.outputs.runs }}"
+          EVAL_CADENCE: ${{ inputs.cadence != 'none' && inputs.cadence || '' }}
+        run: |
+          if [ "${{ inputs.commit_results }}" = "none" ] && { [ -z "$BRAINTRUST_API_KEY" ] || [ -z "$BRAINTRUST_PROJECT_ID" ]; }; then
+            echo "::error::Braintrust credentials are required when commit_results is none."
+            exit 1
+          fi
+          pnpm upload-braintrust --runs "${{ needs.prepare.outputs.runs }}"
 
       - name: Upload exported results
         if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
@@ -545,13 +577,10 @@ jobs:
           retention-days: 7
 
       - name: Commit exported results to branch
-        # PR runs commit to the PR head branch. Manual dispatch commits to the
-        # selected branch only when commit_to_branch is enabled. Scheduled runs
-        # go through a PR instead (see "Create results pull request" below):
-        # main's branch protection rejects a direct push.
+        # PR runs commit to the PR head branch.
         if: >-
           github.event_name == 'pull_request' ||
-          (github.event_name == 'workflow_dispatch' && inputs.commit_to_branch)
+          (github.event_name != 'pull_request' && inputs.commit_results == 'current-branch')
         shell: bash
         run: |
           set -euo pipefail
@@ -575,8 +604,7 @@ jobs:
 
       - name: Create results pull request
         if: >-
-          github.event_name == 'schedule' ||
-          (github.event_name == 'workflow_dispatch' && !inputs.commit_to_branch && inputs.results_pr)
+          github.event_name != 'pull_request' && inputs.commit_results == 'results-pr'
         id: cpr
         uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
         with:
diff --git a/.github/workflows/regression-eval-schedule.yml b/.github/workflows/regression-eval-schedule.yml
new file mode 100644
index 00000000..3d5633e5
--- /dev/null
+++ b/.github/workflows/regression-eval-schedule.yml
@@ -0,0 +1,18 @@
+name: Regression eval schedule
+
+on:
+  schedule:
+    - cron: '15 6 * * *'
+
+permissions:
+  contents: write
+  pull-requests: write
+  actions: read
+
+jobs:
+  run:
+    uses: ./.github/workflows/eval-refresh.yml
+    secrets: inherit
+    with:
+      suite: regression
+      merge: false
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7c710e11..b62d9b89 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -56,7 +56,7 @@ You have a few options to run evals in CI:
 
 - Add the `run-evals-changed` label to your PR to refresh only the `evals/` changed in that PR and commit merged results directly to your branch.
 - Add the `run-evals` label to run every benchmark eval across the experiments in `experiments/benchmark/`. Use this when a change can affect results broadly, such as framework changes.
-- Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow manually to target any branch and choose specific evals, experiments, or other options. It can commit results directly to the selected branch or open a separate results PR.
+- Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow manually to target any branch and choose specific evals, experiments, or other options. Set `commit_results` to `current-branch`, `results-pr`, or `none` to choose where to commit the results JSON.
 
 Include refreshed results for PRs with new/changed evals so a reviewer can inspect them in Braintrust.
 
@@ -114,7 +114,7 @@ Common workflows:
 - **Add or change a CLI eval.** Add the scenario under `evals/cli/<id>/` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview.
 - **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version`/`cli_next_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run.
 - **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories.
-- **Scheduled runs.** The CLI team's schedule dispatches [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) with `suite: cli` and `results_pr: false`, so its results go to the Evals project in Braintrust only, tagged `cadence: daily|weekly`, and don't update `cli-eval-results.json`.
+- **Scheduled runs.** The CLI team's schedule calls [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml). Results go to Braintrust with `cadence` set to `daily` or `weekly`.
   - Daily, 04:20 UTC: the six `codex-gpt-6-luna-cli-*` arms.
   - Weekly, Monday 01:20 UTC: the ten-arm matrix (five models on stable and next).
   - Edit the cadence and experiment lists in `.github/workflows/cli-eval-schedule.yml`.
diff --git a/apps/framework/scripts/upload-braintrust.test.ts b/apps/framework/scripts/upload-braintrust.test.ts
index 7ee98b50..09669c74 100644
--- a/apps/framework/scripts/upload-braintrust.test.ts
+++ b/apps/framework/scripts/upload-braintrust.test.ts
@@ -141,21 +141,10 @@ describe('ciRunUrl', () => {
 });
 
 describe('cadence', () => {
-  afterEach(() => vi.unstubAllEnvs());
-
-  it('reports the dispatching schedule when set', () => {
-    vi.stubEnv('EVAL_CADENCE', ' weekly ');
-    expect(cadence()).toEqual({ cadence: 'weekly' });
-  });
-
-  it('is omitted when unset, empty, or whitespace', () => {
-    vi.stubEnv('EVAL_CADENCE', '');
-    expect(cadence()).toEqual({});
-    vi.stubEnv('EVAL_CADENCE', '  ');
-    expect(cadence()).toEqual({});
-    vi.unstubAllEnvs();
-    delete process.env.EVAL_CADENCE;
-    expect(cadence()).toEqual({});
+  it('records the cadence when present', () => {
+    expect(cadence('daily')).toEqual({ cadence: 'daily' });
+    expect(cadence(undefined)).toEqual({});
+    expect(cadence('')).toEqual({});
   });
 });
 
diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts
index 1c25c680..bc1ad341 100644
--- a/apps/framework/scripts/upload-braintrust.ts
+++ b/apps/framework/scripts/upload-braintrust.ts
@@ -317,8 +317,7 @@ export function ciRunUrl() {
     : {};
 }
 
-export function cadence() {
-  const value = process.env.EVAL_CADENCE?.trim();
+export function cadence(value: string | undefined) {
   return value ? { cadence: value } : {};
 }
 
@@ -764,6 +763,7 @@ async function main() {
     return;
   }
 
+  const runCadence = cadence(process.env.EVAL_CADENCE);
   const experimentMetadata = await loadExperimentMetadata();
   const byExperiment = await collectRows(experimentMetadata);
   if (byExperiment.size === 0) {
@@ -812,7 +812,7 @@ async function main() {
         ...(info?.branch ? { branch: info.branch } : {}),
         ...prUrl(),
         ...ciRunUrl(),
-        ...cadence(),
+        ...runCadence,
       },
     });
 

From 74199cca8fd2e6eade44ce2213622ce6c2855786 Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Fri, 6 Nov 2026 15:06:27 -0500
Subject: [PATCH 5/7] refactor: simplify cli eval schedules

---
 .github/workflows/cli-eval-schedule.yml        |  5 +----
 .github/workflows/eval-refresh.yml             | 18 ------------------
 CONTRIBUTING.md                                |  4 ++--
 .../scripts/upload-braintrust.test.ts          |  9 ---------
 apps/framework/scripts/upload-braintrust.ts    |  6 ------
 5 files changed, 3 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/cli-eval-schedule.yml b/.github/workflows/cli-eval-schedule.yml
index 64cc9ebe..f9fd2c6d 100644
--- a/.github/workflows/cli-eval-schedule.yml
+++ b/.github/workflows/cli-eval-schedule.yml
@@ -27,7 +27,6 @@ jobs:
     with:
       suite: cli
       merge: false
-      cadence: daily
       commit_results: none
       experiments: >-
         codex-gpt-6-luna-cli-pinned,
@@ -44,8 +43,8 @@ jobs:
     with:
       suite: cli
       merge: false
-      cadence: weekly
       commit_results: none
+      # Luna runs daily, including Monday.
       experiments: >-
         claude-code-sonnet-5.5-cli-stable,
         claude-code-sonnet-5.5-cli-next,
@@ -53,7 +52,5 @@ jobs:
         claude-code-opus-5.5-cli-next,
         codex-gpt-6.1-sol-cli-stable,
         codex-gpt-6.1-sol-cli-next,
-        codex-gpt-6-luna-cli-stable,
-        codex-gpt-6-luna-cli-next,
         opencode-kimi-k3-cli-stable,
         opencode-kimi-k3-cli-next
diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index c3ee939a..b177f12d 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -58,14 +58,6 @@ on:
         type: string
         required: false
         default: ""
-      cadence:
-        description: "Schedule recorded on each Braintrust experiment"
-        type: choice
-        options:
-          - none
-          - daily
-          - weekly
-        default: none
   workflow_call:
     inputs:
       experiments: *experiments
@@ -78,9 +70,6 @@ on:
       cli_stable_version: *cli_stable_version
       cli_beta_version: *cli_beta_version
       cli_next_version: *cli_next_version
-      cadence:
-        type: string
-        default: ""
       commit_results:
         type: string
         default: results-pr
@@ -131,7 +120,6 @@ jobs:
         id: inputs
         shell: bash
         env:
-          EVAL_CADENCE: ${{ inputs.cadence != 'none' && inputs.cadence || '' }}
           COMMIT_RESULTS: ${{ inputs.commit_results }}
         run: |
           set -euo pipefail
@@ -143,11 +131,6 @@ jobs:
             esac
           fi
 
-          case "$EVAL_CADENCE" in
-            ""|daily|weekly) ;;
-            *) echo "::error::Invalid cadence: $EVAL_CADENCE"; exit 1 ;;
-          esac
-
           if [ "${{ github.event_name }}" != "pull_request" ]; then
             experiments_override="${{ inputs.experiments }}"
             eval_id="${{ inputs.eval }}"
@@ -559,7 +542,6 @@ jobs:
         env:
           BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
           BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}
-          EVAL_CADENCE: ${{ inputs.cadence != 'none' && inputs.cadence || '' }}
         run: |
           if [ "${{ inputs.commit_results }}" = "none" ] && { [ -z "$BRAINTRUST_API_KEY" ] || [ -z "$BRAINTRUST_PROJECT_ID" ]; }; then
             echo "::error::Braintrust credentials are required when commit_results is none."
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index b62d9b89..f87f9264 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -114,9 +114,9 @@ Common workflows:
 - **Add or change a CLI eval.** Add the scenario under `evals/cli/<id>/` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview.
 - **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version`/`cli_next_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run.
 - **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories.
-- **Scheduled runs.** The CLI team's schedule calls [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml). Results go to Braintrust with `cadence` set to `daily` or `weekly`.
+- **Scheduled runs.** The CLI team's schedule calls [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml). Results go to Braintrust.
   - Daily, 04:20 UTC: the six `codex-gpt-6-luna-cli-*` arms.
-  - Weekly, Monday 01:20 UTC: the ten-arm matrix (five models on stable and next).
+  - Weekly, other models on stable and next. See `.github/workflows/cli-eval-schedule.yml` for the schedule and experiment list.
   - Edit the cadence and experiment lists in `.github/workflows/cli-eval-schedule.yml`.
   - The shared nightly no longer runs the cli suite.
 - **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm test`.
diff --git a/apps/framework/scripts/upload-braintrust.test.ts b/apps/framework/scripts/upload-braintrust.test.ts
index 09669c74..322c178d 100644
--- a/apps/framework/scripts/upload-braintrust.test.ts
+++ b/apps/framework/scripts/upload-braintrust.test.ts
@@ -1,6 +1,5 @@
 import { afterEach, describe, expect, it, vi } from 'vitest';
 import {
-  cadence,
   ciRunUrl,
   experimentName,
   judgeMetrics,
@@ -140,14 +139,6 @@ describe('ciRunUrl', () => {
   });
 });
 
-describe('cadence', () => {
-  it('records the cadence when present', () => {
-    expect(cadence('daily')).toEqual({ cadence: 'daily' });
-    expect(cadence(undefined)).toEqual({});
-    expect(cadence('')).toEqual({});
-  });
-});
-
 describe('runViewUrl', () => {
   it('filters the experiments list by run id', () => {
     const url = runViewUrl(
diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts
index bc1ad341..6f4e617d 100644
--- a/apps/framework/scripts/upload-braintrust.ts
+++ b/apps/framework/scripts/upload-braintrust.ts
@@ -317,10 +317,6 @@ export function ciRunUrl() {
     : {};
 }
 
-export function cadence(value: string | undefined) {
-  return value ? { cadence: value } : {};
-}
-
 async function collectRows(
   experimentMetadata: Map<string, ExperimentMetadata>
 ): Promise<Map<string, ExperimentGroup>> {
@@ -763,7 +759,6 @@ async function main() {
     return;
   }
 
-  const runCadence = cadence(process.env.EVAL_CADENCE);
   const experimentMetadata = await loadExperimentMetadata();
   const byExperiment = await collectRows(experimentMetadata);
   if (byExperiment.size === 0) {
@@ -812,7 +807,6 @@ async function main() {
         ...(info?.branch ? { branch: info.branch } : {}),
         ...prUrl(),
         ...ciRunUrl(),
-        ...runCadence,
       },
     });
 

From 92d57cbab070b8069ffd49919fd191aa249c1dad Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Fri, 6 Nov 2026 15:24:38 -0500
Subject: [PATCH 6/7] docs: explain eval schedules

---
 CONTRIBUTING.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f87f9264..dcb03762 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -58,6 +58,8 @@ You have a few options to run evals in CI:
 - Add the `run-evals` label to run every benchmark eval across the experiments in `experiments/benchmark/`. Use this when a change can affect results broadly, such as framework changes.
 - Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow manually to target any branch and choose specific evals, experiments, or other options. Set `commit_results` to `current-branch`, `results-pr`, or `none` to choose where to commit the results JSON.
 
+To schedule a suite, add a team-owned workflow that calls [eval-refresh.yml](.github/workflows/eval-refresh.yml) with the suite and optional experiments, and add the workflow to [CODEOWNERS](.github/CODEOWNERS). See existing schedules in [.github/workflows](.github/workflows/) for reference.
+
 Include refreshed results for PRs with new/changed evals so a reviewer can inspect them in Braintrust.
 
 ## Publishing and consuming results

From 24b07021aab323a1cec2f625e78b30ee636703ea Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Fri, 6 Nov 2026 15:27:23 -0500
Subject: [PATCH 7/7] docs: tweak comment

---
 .github/workflows/cli-eval-schedule.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cli-eval-schedule.yml b/.github/workflows/cli-eval-schedule.yml
index f9fd2c6d..c6a454a3 100644
--- a/.github/workflows/cli-eval-schedule.yml
+++ b/.github/workflows/cli-eval-schedule.yml
@@ -44,7 +44,7 @@ jobs:
       suite: cli
       merge: false
       commit_results: none
-      # Luna runs daily, including Monday.
+      # Luna experiments already run daily, so they're excluded from the weekly schedule to avoid duplicate runs.
       experiments: >-
         claude-code-sonnet-5.5-cli-stable,
         claude-code-sonnet-5.5-cli-next,