feat(cli): run the cli suite on its own daily and weekly schedule

.github/CODEOWNERS

@@ -8,4 +8,5 @@
 
 /evals/cli/ @supabase/cli
 /experiments/cli/ @supabase/cli
+/.github/workflows/cli-eval-schedule.yml @supabase/cli
 /apps/web/src/data/cli-eval-results.json @supabase/cli

.github/workflows/cli-eval-schedule.yml

@@ -0,0 +1,76 @@
+# The CLI team owns this cadence and reuses eval-refresh's run and publish pipeline.
+# Results go to Braintrust only; no cli-eval-results.json PR is opened.
+name: CLI eval schedule
+
+on:
+  schedule:
+    - cron: '20 4 * * *'
+    # CADENCE below picks the weekly list by matching this string.
+    - cron: '20 1 * * 1'
+  workflow_dispatch:
+    inputs:
+      cadence:
+        description: "Which experiment list to run"
+        type: choice
+        options:
+          - daily
+          - weekly
+        default: daily
+
+permissions:
+  actions: write
+
+env:
+  DAILY_EXPERIMENTS: >-
+    codex-gpt-6-luna-cli-pinned,
+    codex-gpt-6-luna-cli-stable,
+    codex-gpt-6-luna-cli-beta,
+    codex-gpt-6-luna-cli-next,
+    codex-gpt-6-luna-cli-nodaemon,
+    codex-gpt-6-luna-cli-absent
+  WEEKLY_EXPERIMENTS: >-
+    claude-code-sonnet-5.5-cli-stable,
+    claude-code-sonnet-5.5-cli-next,
+    claude-code-opus-5.5-cli-stable,
+    claude-code-opus-5.5-cli-next,
+    codex-gpt-6.1-sol-cli-stable,
+    codex-gpt-6.1-sol-cli-next,
+    codex-gpt-6-luna-cli-stable,
+    codex-gpt-6-luna-cli-next,
+    opencode-kimi-k3-cli-stable,
+    opencode-kimi-k3-cli-next
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-26.04
+    timeout-minutes: 5
+    steps:
+      - name: Dispatch eval-refresh
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          CADENCE: ${{ github.event_name == 'workflow_dispatch' && inputs.cadence || github.event.schedule == '20 1 * * 1' && 'weekly' || 'daily' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          case "$CADENCE" in
+            daily) experiments="$DAILY_EXPERIMENTS" ;;
+            weekly) experiments="$WEEKLY_EXPERIMENTS" ;;
+            *)
+              echo "::error::Unknown cadence: $CADENCE"
+              exit 1
+              ;;
+          esac
+
+          gh workflow run eval-refresh.yml \
+            --ref "$GITHUB_REF_NAME" \
+            -f suite=cli \
+            -f cadence="$CADENCE" \
+            -f experiments="$experiments" \
+            -f runs=3 \
+            -f timeout_sec=720 \
+            -f sandbox_concurrency=250 \
+            -f results_pr=false
+
+          echo "Dispatched eval-refresh for the $CADENCE cadence: $experiments" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/eval-refresh.yml

@@ -37,6 +37,11 @@ on:
         type: boolean
         required: false
         default: false
+      results_pr:
+        description: "Open a PR with the exported results JSON (off for Braintrust-only callers such as cli-eval-schedule.yml)"
+        type: boolean
+        required: false
+        default: true
       cli_stable_version:
         description: "Pin the cli suite's stable CLI version instead of resolving npm's latest dist-tag (blank to resolve)"
         required: false
@@ -49,6 +54,11 @@ on:
         description: "Pin the cli suite's next CLI version instead of resolving npm's next dist-tag (blank to resolve)"
         required: false
         default: ""
+      cadence:
+        description: "Schedule that dispatched this run (daily or weekly); recorded on each Braintrust experiment"
+        type: string
+        required: false
+        default: ""
   schedule:
     - cron: '15 6 * * *'
   pull_request:
@@ -118,7 +128,8 @@ jobs:
           elif [ "${{ github.event_name }}" = "schedule" ]; then
             experiments_override=""
             eval_id=""
-            suite="regression,cli"
+            # The cli suite runs on its own cadence in cli-eval-schedule.yml.
+            suite="regression"
             runs="3"
             timeout_sec="720"
             sandbox_concurrency="250"
@@ -516,11 +527,12 @@ jobs:
           echo "PR_URLS=$urls" >> "$GITHUB_ENV"
 
       - name: Upload results to Braintrust
-        # A Braintrust outage shouldn't block publishing results.
-        continue-on-error: true
+        # A Braintrust outage shouldn't block publishing results, but with no results PR it is the only record.
+        continue-on-error: ${{ github.event_name != 'workflow_dispatch' || inputs.results_pr }}
         env:
           BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
           BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}
+          EVAL_CADENCE: ${{ inputs.cadence }}
         run: pnpm upload-braintrust --runs "${{ needs.prepare.outputs.runs }}"
 
       - name: Upload exported results
@@ -564,7 +576,7 @@ jobs:
       - name: Create results pull request
         if: >-
           github.event_name == 'schedule' ||
-          (github.event_name == 'workflow_dispatch' && !inputs.commit_to_branch)
+          (github.event_name == 'workflow_dispatch' && !inputs.commit_to_branch && inputs.results_pr)
         id: cpr
         uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
         with:

CONTRIBUTING.md

@@ -102,16 +102,21 @@ Common workflows:
 
 ## CLI evals
 
-The CLI team owns `evals/cli/` and its results. CLI evals run on `codex-gpt-6-luna-cli-{pinned,stable,beta,next,nodaemon,absent}` under `experiments/cli/`: `pinned` runs the repo's pinned CLI version, `stable`/`beta`/`next` install the latest stable, beta or next CLI, and `nodaemon`/`absent` additionally force Docker-less sandboxes — comparing the same scenario across CLI environments.
+The CLI team owns `evals/cli/` and its results. CLI evals run on `codex-gpt-6-luna-cli-{pinned,stable,beta,next,nodaemon,absent}` under `experiments/cli/`: `pinned` runs the repo's pinned CLI version, `stable`/`beta`/`next` install the latest stable, beta or next CLI, and `nodaemon`/`absent` additionally force Docker-less sandboxes — comparing the same scenario across CLI environments. The `<model>-cli-{stable,next}` arms run the same scenario on the stable and next CLI across `claude-code-sonnet-5.5`, `claude-code-opus-5.5`, `codex-gpt-6.1-sol` and `opencode-kimi-k3`, comparing models rather than environments.
 
 Which evals each arm picks up:
 
-- **pinned, stable, beta, next** run every `interface: cli` eval that isn't `hostedProject: true`.
+- **pinned, stable, beta, next** run every `interface: cli` eval that isn't `hostedProject: true`. The `<model>-cli-{stable,next}` matrix arms pick up the same evals as `stable` and `next`.
 - **nodaemon, absent** additionally only run evals that also set `needsDocker: false` and `projectRunning: false` — the harness cannot pre-start a stack, or link a hosted project, without Docker.
 
 Common workflows:
 
 - **Add or change a CLI eval.** Add the scenario under `evals/cli/<id>/` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview.
 - **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version`/`cli_next_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run.
 - **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories.
+- **Scheduled runs.** The CLI team's schedule dispatches [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) with `suite: cli` and `results_pr: false`, so its results go to the Evals project in Braintrust only, tagged `cadence: daily|weekly`, and don't update `cli-eval-results.json`.
+  - Daily, 04:20 UTC: the six `codex-gpt-6-luna-cli-*` arms.
+  - Weekly, Monday 01:20 UTC: the ten-arm matrix (five models on stable and next).
+  - Edit the cadence and experiment lists in `.github/workflows/cli-eval-schedule.yml`.
+  - The shared nightly no longer runs the cli suite.
 - **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm test`.

apps/framework/scripts/upload-braintrust.test.ts

@@ -1,5 +1,6 @@
 import { afterEach, describe, expect, it, vi } from 'vitest';
 import {
+  cadence,
   ciRunUrl,
   experimentName,
   judgeMetrics,
@@ -139,6 +140,25 @@ describe('ciRunUrl', () => {
   });
 });
 
+describe('cadence', () => {
+  afterEach(() => vi.unstubAllEnvs());
+
+  it('reports the dispatching schedule when set', () => {
+    vi.stubEnv('EVAL_CADENCE', ' weekly ');
+    expect(cadence()).toEqual({ cadence: 'weekly' });
+  });
+
+  it('is omitted when unset, empty, or whitespace', () => {
+    vi.stubEnv('EVAL_CADENCE', '');
+    expect(cadence()).toEqual({});
+    vi.stubEnv('EVAL_CADENCE', '  ');
+    expect(cadence()).toEqual({});
+    vi.unstubAllEnvs();
+    delete process.env.EVAL_CADENCE;
+    expect(cadence()).toEqual({});
+  });
+});
+
 describe('runViewUrl', () => {
   it('filters the experiments list by run id', () => {
     const url = runViewUrl(

apps/framework/scripts/upload-braintrust.ts

@@ -317,6 +317,11 @@ export function ciRunUrl() {
     : {};
 }
 
+export function cadence() {
+  const value = process.env.EVAL_CADENCE?.trim();
+  return value ? { cadence: value } : {};
+}
+
 async function collectRows(
   experimentMetadata: Map<string, ExperimentMetadata>
 ): Promise<Map<string, ExperimentGroup>> {
@@ -807,6 +812,7 @@ async function main() {
         ...(info?.branch ? { branch: info.branch } : {}),
         ...prUrl(),
         ...ciRunUrl(),
+        ...cadence(),
       },
     });
 

experiments/cli/claude-code-opus-5.5-cli-next.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { claudeCodeOpus55 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...claudeCodeOpus55,
+  localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/claude-code-opus-5.5-cli-stable.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { claudeCodeOpus55 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...claudeCodeOpus55,
+  localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/claude-code-sonnet-5.5-cli-next.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { claudeCodeSonnet55 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...claudeCodeSonnet55,
+  localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/claude-code-sonnet-5.5-cli-stable.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { claudeCodeSonnet55 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...claudeCodeSonnet55,
+  localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/codex-gpt-6.1-sol-cli-next.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { codexGpt61Sol } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...codexGpt61Sol,
+  localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/codex-gpt-6.1-sol-cli-stable.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { codexGpt61Sol } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...codexGpt61Sol,
+  localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/opencode-kimi-k3-cli-next.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { opencodeKimiK3 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...opencodeKimiK3,
+  localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/opencode-kimi-k3-cli-stable.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { opencodeKimiK3 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...opencodeKimiK3,
+  localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});