diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f63d16b1..aa9e5a2a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,4 +8,5 @@ /evals/cli/ @supabase/cli /experiments/cli/ @supabase/cli +/.github/workflows/cli-eval-schedule.yml @supabase/cli /apps/web/src/data/cli-eval-results.json @supabase/cli diff --git a/.github/workflows/cli-eval-schedule.yml b/.github/workflows/cli-eval-schedule.yml new file mode 100644 index 00000000..c6a454a3 --- /dev/null +++ b/.github/workflows/cli-eval-schedule.yml @@ -0,0 +1,56 @@ +name: CLI eval schedule + +on: + schedule: + - cron: '20 4 * * *' + - cron: '20 1 * * 1' + workflow_dispatch: + inputs: + cadence: + description: "Which experiment list to run" + type: choice + options: + - daily + - weekly + default: daily + +permissions: + contents: write + pull-requests: write + actions: read + +jobs: + daily: + if: github.event.schedule == '20 4 * * *' || (github.event_name == 'workflow_dispatch' && inputs.cadence == 'daily') + uses: ./.github/workflows/eval-refresh.yml + secrets: inherit + with: + suite: cli + merge: false + commit_results: none + experiments: >- + codex-gpt-6-luna-cli-pinned, + codex-gpt-6-luna-cli-stable, + codex-gpt-6-luna-cli-beta, + codex-gpt-6-luna-cli-next, + codex-gpt-6-luna-cli-nodaemon, + codex-gpt-6-luna-cli-absent + + weekly: + if: github.event.schedule == '20 1 * * 1' || (github.event_name == 'workflow_dispatch' && inputs.cadence == 'weekly') + uses: ./.github/workflows/eval-refresh.yml + secrets: inherit + with: + suite: cli + merge: false + commit_results: none + # Luna experiments already run daily, so they're excluded from the weekly schedule to avoid duplicate runs. + experiments: >- + claude-code-sonnet-5.5-cli-stable, + claude-code-sonnet-5.5-cli-next, + claude-code-opus-5.5-cli-stable, + claude-code-opus-5.5-cli-next, + codex-gpt-6.1-sol-cli-stable, + codex-gpt-6.1-sol-cli-next, + opencode-kimi-k3-cli-stable, + opencode-kimi-k3-cli-next diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 24c16b7d..b177f12d 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -3,54 +3,76 @@ name: Refresh eval results on: workflow_dispatch: inputs: - experiments: + experiments: &experiments description: "Comma-separated experiment names to run within each selected eval suite" + type: string required: false default: "" - eval: + eval: &eval description: "Comma-separated eval ids to run (overrides suite)" + type: string required: false default: "" - suite: + suite: &suite description: "Comma-separated suites to run (evals/ with experiments/)" + type: string required: false default: "" - runs: + runs: &runs description: "Independent scored runs per experiment/eval pair" - required: true + type: string default: "3" - timeout_sec: + timeout_sec: &timeout_sec description: "Timeout per run in seconds" - required: true + type: string default: "720" - sandbox_concurrency: + sandbox_concurrency: &sandbox_concurrency description: "Maximum Vercel Sandboxes running at once" - required: true + type: string default: "250" - merge: + merge: &merge description: "Merge into existing results instead of overwriting (graft new experiment/eval pairs)" type: boolean required: false default: true - commit_to_branch: - description: "Commit exported results to the dispatched branch instead of opening a PR" - type: boolean - required: false - default: false - cli_stable_version: + commit_results: + description: "Where to commit exported results JSON" + type: choice + options: + - results-pr + - current-branch + - none + default: results-pr + cli_stable_version: &cli_stable_version description: "Pin the cli suite's stable CLI version instead of resolving npm's latest dist-tag (blank to resolve)" + type: string required: false default: "" - cli_beta_version: + cli_beta_version: &cli_beta_version description: "Pin the cli suite's beta CLI version instead of resolving npm's beta dist-tag (blank to resolve)" + type: string required: false default: "" - cli_next_version: + cli_next_version: &cli_next_version description: "Pin the cli suite's next CLI version instead of resolving npm's next dist-tag (blank to resolve)" + type: string required: false default: "" - schedule: - - cron: '15 6 * * *' + workflow_call: + inputs: + experiments: *experiments + eval: *eval + suite: *suite + runs: *runs + timeout_sec: *timeout_sec + sandbox_concurrency: *sandbox_concurrency + merge: *merge + cli_stable_version: *cli_stable_version + cli_beta_version: *cli_beta_version + cli_next_version: *cli_next_version + commit_results: + type: string + default: results-pr pull_request: # Run whenever a PR carrying the run-evals label is opened, pushed to, or # receives the run-evals label. The job-level `if` gates on those cases. @@ -63,9 +85,9 @@ permissions: concurrency: # The results app's pushes get a unique group so they can't cancel the run that pushed them. - group: eval-refresh-${{ github.event.sender.id == 291952723 && github.run_id || github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} + group: eval-refresh-${{ github.workflow }}-${{ github.event.sender.id == 291952723 && github.run_id || github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} # Supersede an in-flight run when a new commit is pushed to the same PR, but - # never cancel a workflow_dispatch run (those open the refresh PR). + # never cancel a workflow_dispatch run. cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: @@ -97,10 +119,19 @@ jobs: - name: Prepare inputs id: inputs shell: bash + env: + COMMIT_RESULTS: ${{ inputs.commit_results }} run: | set -euo pipefail - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + if [ "${{ github.event_name }}" != "pull_request" ]; then + case "$COMMIT_RESULTS" in + results-pr|current-branch|none) ;; + *) echo "::error::Invalid commit_results: $COMMIT_RESULTS"; exit 1 ;; + esac + fi + + if [ "${{ github.event_name }}" != "pull_request" ]; then experiments_override="${{ inputs.experiments }}" eval_id="${{ inputs.eval }}" suite="${{ inputs.suite }}" @@ -115,16 +146,6 @@ jobs: echo "::error::Set suite or eval to choose which evals to run." exit 1 fi - elif [ "${{ github.event_name }}" = "schedule" ]; then - experiments_override="" - eval_id="" - suite="regression,cli" - runs="3" - timeout_sec="720" - sandbox_concurrency="250" - cli_stable_version="" - cli_beta_version="" - cli_next_version="" else experiments_override="" eval_id="" @@ -516,12 +537,17 @@ jobs: echo "PR_URLS=$urls" >> "$GITHUB_ENV" - name: Upload results to Braintrust - # A Braintrust outage shouldn't block publishing results. - continue-on-error: true + # A Braintrust outage shouldn't block publishing results, but with no results PR it is the only record. + continue-on-error: ${{ github.event_name == 'pull_request' || inputs.commit_results != 'none' }} env: BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }} - run: pnpm upload-braintrust --runs "${{ needs.prepare.outputs.runs }}" + run: | + if [ "${{ inputs.commit_results }}" = "none" ] && { [ -z "$BRAINTRUST_API_KEY" ] || [ -z "$BRAINTRUST_PROJECT_ID" ]; }; then + echo "::error::Braintrust credentials are required when commit_results is none." + exit 1 + fi + pnpm upload-braintrust --runs "${{ needs.prepare.outputs.runs }}" - name: Upload exported results if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' @@ -533,13 +559,10 @@ jobs: retention-days: 7 - name: Commit exported results to branch - # PR runs commit to the PR head branch. Manual dispatch commits to the - # selected branch only when commit_to_branch is enabled. Scheduled runs - # go through a PR instead (see "Create results pull request" below): - # main's branch protection rejects a direct push. + # PR runs commit to the PR head branch. if: >- github.event_name == 'pull_request' || - (github.event_name == 'workflow_dispatch' && inputs.commit_to_branch) + (github.event_name != 'pull_request' && inputs.commit_results == 'current-branch') shell: bash run: | set -euo pipefail @@ -563,8 +586,7 @@ jobs: - name: Create results pull request if: >- - github.event_name == 'schedule' || - (github.event_name == 'workflow_dispatch' && !inputs.commit_to_branch) + github.event_name != 'pull_request' && inputs.commit_results == 'results-pr' id: cpr uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: diff --git a/.github/workflows/regression-eval-schedule.yml b/.github/workflows/regression-eval-schedule.yml new file mode 100644 index 00000000..3d5633e5 --- /dev/null +++ b/.github/workflows/regression-eval-schedule.yml @@ -0,0 +1,18 @@ +name: Regression eval schedule + +on: + schedule: + - cron: '15 6 * * *' + +permissions: + contents: write + pull-requests: write + actions: read + +jobs: + run: + uses: ./.github/workflows/eval-refresh.yml + secrets: inherit + with: + suite: regression + merge: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0add5ba7..dcb03762 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,7 +56,9 @@ You have a few options to run evals in CI: - Add the `run-evals-changed` label to your PR to refresh only the `evals/` changed in that PR and commit merged results directly to your branch. - Add the `run-evals` label to run every benchmark eval across the experiments in `experiments/benchmark/`. Use this when a change can affect results broadly, such as framework changes. -- Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow manually to target any branch and choose specific evals, experiments, or other options. It can commit results directly to the selected branch or open a separate results PR. +- Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow manually to target any branch and choose specific evals, experiments, or other options. Set `commit_results` to `current-branch`, `results-pr`, or `none` to choose where to commit the results JSON. + +To schedule a suite, add a team-owned workflow that calls [eval-refresh.yml](.github/workflows/eval-refresh.yml) with the suite and optional experiments, and add the workflow to [CODEOWNERS](.github/CODEOWNERS). See existing schedules in [.github/workflows](.github/workflows/) for reference. Include refreshed results for PRs with new/changed evals so a reviewer can inspect them in Braintrust. @@ -102,11 +104,11 @@ Common workflows: ## CLI evals -The CLI team owns `evals/cli/` and its results. CLI evals run on `codex-gpt-6-luna-cli-{pinned,stable,beta,next,nodaemon,absent}` under `experiments/cli/`: `pinned` runs the repo's pinned CLI version, `stable`/`beta`/`next` install the latest stable, beta or next CLI, and `nodaemon`/`absent` additionally force Docker-less sandboxes — comparing the same scenario across CLI environments. +The CLI team owns `evals/cli/` and its results. CLI evals run on `codex-gpt-6-luna-cli-{pinned,stable,beta,next,nodaemon,absent}` under `experiments/cli/`: `pinned` runs the repo's pinned CLI version, `stable`/`beta`/`next` install the latest stable, beta or next CLI, and `nodaemon`/`absent` additionally force Docker-less sandboxes — comparing the same scenario across CLI environments. The `-cli-{stable,next}` arms run the same scenario on the stable and next CLI across `claude-code-sonnet-5.5`, `claude-code-opus-5.5`, `codex-gpt-6.1-sol` and `opencode-kimi-k3`, comparing models rather than environments. Which evals each arm picks up: -- **pinned, stable, beta, next** run every `interface: cli` eval that isn't `hostedProject: true`. +- **pinned, stable, beta, next** run every `interface: cli` eval that isn't `hostedProject: true`. The `-cli-{stable,next}` matrix arms pick up the same evals as `stable` and `next`. - **nodaemon, absent** additionally only run evals that also set `needsDocker: false` and `projectRunning: false` — the harness cannot pre-start a stack, or link a hosted project, without Docker. Common workflows: @@ -114,4 +116,9 @@ Common workflows: - **Add or change a CLI eval.** Add the scenario under `evals/cli//` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview. - **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version`/`cli_next_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run. - **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories. +- **Scheduled runs.** The CLI team's schedule calls [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml). Results go to Braintrust. + - Daily, 04:20 UTC: the six `codex-gpt-6-luna-cli-*` arms. + - Weekly, other models on stable and next. See `.github/workflows/cli-eval-schedule.yml` for the schedule and experiment list. + - Edit the cadence and experiment lists in `.github/workflows/cli-eval-schedule.yml`. + - The shared nightly no longer runs the cli suite. - **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm test`. diff --git a/experiments/cli/claude-code-opus-5.5-cli-next.experiment.ts b/experiments/cli/claude-code-opus-5.5-cli-next.experiment.ts new file mode 100644 index 00000000..62d9b209 --- /dev/null +++ b/experiments/cli/claude-code-opus-5.5-cli-next.experiment.ts @@ -0,0 +1,11 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { claudeCodeOpus55 } from '../presets.js'; +import { EVAL_ENV } from './lib/eval-env.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...claudeCodeOpus55, + localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }), + skipEval: skipUnlessCli, +}); diff --git a/experiments/cli/claude-code-opus-5.5-cli-stable.experiment.ts b/experiments/cli/claude-code-opus-5.5-cli-stable.experiment.ts new file mode 100644 index 00000000..c1f15941 --- /dev/null +++ b/experiments/cli/claude-code-opus-5.5-cli-stable.experiment.ts @@ -0,0 +1,11 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { claudeCodeOpus55 } from '../presets.js'; +import { EVAL_ENV } from './lib/eval-env.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...claudeCodeOpus55, + localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }), + skipEval: skipUnlessCli, +}); diff --git a/experiments/cli/claude-code-sonnet-5.5-cli-next.experiment.ts b/experiments/cli/claude-code-sonnet-5.5-cli-next.experiment.ts new file mode 100644 index 00000000..68204be6 --- /dev/null +++ b/experiments/cli/claude-code-sonnet-5.5-cli-next.experiment.ts @@ -0,0 +1,11 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { claudeCodeSonnet55 } from '../presets.js'; +import { EVAL_ENV } from './lib/eval-env.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...claudeCodeSonnet55, + localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }), + skipEval: skipUnlessCli, +}); diff --git a/experiments/cli/claude-code-sonnet-5.5-cli-stable.experiment.ts b/experiments/cli/claude-code-sonnet-5.5-cli-stable.experiment.ts new file mode 100644 index 00000000..fdd0b12d --- /dev/null +++ b/experiments/cli/claude-code-sonnet-5.5-cli-stable.experiment.ts @@ -0,0 +1,11 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { claudeCodeSonnet55 } from '../presets.js'; +import { EVAL_ENV } from './lib/eval-env.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...claudeCodeSonnet55, + localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }), + skipEval: skipUnlessCli, +}); diff --git a/experiments/cli/codex-gpt-6.1-sol-cli-next.experiment.ts b/experiments/cli/codex-gpt-6.1-sol-cli-next.experiment.ts new file mode 100644 index 00000000..92852295 --- /dev/null +++ b/experiments/cli/codex-gpt-6.1-sol-cli-next.experiment.ts @@ -0,0 +1,11 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { codexGpt61Sol } from '../presets.js'; +import { EVAL_ENV } from './lib/eval-env.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...codexGpt61Sol, + localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }), + skipEval: skipUnlessCli, +}); diff --git a/experiments/cli/codex-gpt-6.1-sol-cli-stable.experiment.ts b/experiments/cli/codex-gpt-6.1-sol-cli-stable.experiment.ts new file mode 100644 index 00000000..654e2892 --- /dev/null +++ b/experiments/cli/codex-gpt-6.1-sol-cli-stable.experiment.ts @@ -0,0 +1,11 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { codexGpt61Sol } from '../presets.js'; +import { EVAL_ENV } from './lib/eval-env.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...codexGpt61Sol, + localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }), + skipEval: skipUnlessCli, +}); diff --git a/experiments/cli/opencode-kimi-k3-cli-next.experiment.ts b/experiments/cli/opencode-kimi-k3-cli-next.experiment.ts new file mode 100644 index 00000000..d5918e02 --- /dev/null +++ b/experiments/cli/opencode-kimi-k3-cli-next.experiment.ts @@ -0,0 +1,11 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { opencodeKimiK3 } from '../presets.js'; +import { EVAL_ENV } from './lib/eval-env.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...opencodeKimiK3, + localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }), + skipEval: skipUnlessCli, +}); diff --git a/experiments/cli/opencode-kimi-k3-cli-stable.experiment.ts b/experiments/cli/opencode-kimi-k3-cli-stable.experiment.ts new file mode 100644 index 00000000..970fb49a --- /dev/null +++ b/experiments/cli/opencode-kimi-k3-cli-stable.experiment.ts @@ -0,0 +1,11 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { opencodeKimiK3 } from '../presets.js'; +import { EVAL_ENV } from './lib/eval-env.js'; +import { skipUnlessCli } from './lib/skip.js'; + +export default defineExperiment({ + ...opencodeKimiK3, + localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }), + skipEval: skipUnlessCli, +});