feat(cli): run the cli suite on its own daily and weekly schedule

.github/CODEOWNERS

@@ -8,4 +8,5 @@
 
 /evals/cli/ @supabase/cli
 /experiments/cli/ @supabase/cli
+/.github/workflows/cli-eval-schedule.yml @supabase/cli
 /apps/web/src/data/cli-eval-results.json @supabase/cli

.github/workflows/cli-eval-schedule.yml

@@ -0,0 +1,56 @@
+name: CLI eval schedule
+
+on:
+  schedule:
+    - cron: '20 4 * * *'
+    - cron: '20 1 * * 1'
+  workflow_dispatch:
+    inputs:
+      cadence:
+        description: "Which experiment list to run"
+        type: choice
+        options:
+          - daily
+          - weekly
+        default: daily
+
+permissions:
+  contents: write
+  pull-requests: write
+  actions: read
+
+jobs:
+  daily:
+    if: github.event.schedule == '20 4 * * *' || (github.event_name == 'workflow_dispatch' && inputs.cadence == 'daily')
+    uses: ./.github/workflows/eval-refresh.yml
+    secrets: inherit
+    with:
+      suite: cli
+      merge: false
+      commit_results: none
+      experiments: >-
+        codex-gpt-6-luna-cli-pinned,
+        codex-gpt-6-luna-cli-stable,
+        codex-gpt-6-luna-cli-beta,
+        codex-gpt-6-luna-cli-next,
+        codex-gpt-6-luna-cli-nodaemon,
+        codex-gpt-6-luna-cli-absent
+
+  weekly:
+    if: github.event.schedule == '20 1 * * 1' || (github.event_name == 'workflow_dispatch' && inputs.cadence == 'weekly')
+    uses: ./.github/workflows/eval-refresh.yml
+    secrets: inherit
+    with:
+      suite: cli
+      merge: false
+      commit_results: none
+      # Luna experiments already run daily, so they're excluded from the weekly schedule to avoid duplicate runs.
+      experiments: >-
+        claude-code-sonnet-5.5-cli-stable,
+        claude-code-sonnet-5.5-cli-next,
+        claude-code-opus-5.5-cli-stable,
+        claude-code-opus-5.5-cli-next,
+        codex-gpt-6.1-sol-cli-stable,
+        codex-gpt-6.1-sol-cli-next,
+        opencode-kimi-k3-cli-stable,
+        opencode-kimi-k3-cli-next

.github/workflows/eval-refresh.yml

@@ -3,54 +3,76 @@ name: Refresh eval results
 on:
   workflow_dispatch:
     inputs:
-      experiments:
+      experiments: &experiments
         description: "Comma-separated experiment names to run within each selected eval suite"
+        type: string
         required: false
         default: ""
-      eval:
+      eval: &eval
         description: "Comma-separated eval ids to run (overrides suite)"
+        type: string
         required: false
         default: ""
-      suite:
+      suite: &suite
         description: "Comma-separated suites to run (evals/<suite> with experiments/<suite>)"
+        type: string
         required: false
         default: ""
-      runs:
+      runs: &runs
         description: "Independent scored runs per experiment/eval pair"
-        required: true
+        type: string
         default: "3"
-      timeout_sec:
+      timeout_sec: &timeout_sec
         description: "Timeout per run in seconds"
-        required: true
+        type: string
         default: "720"
-      sandbox_concurrency:
+      sandbox_concurrency: &sandbox_concurrency
         description: "Maximum Vercel Sandboxes running at once"
-        required: true
+        type: string
         default: "250"
-      merge:
+      merge: &merge
         description: "Merge into existing results instead of overwriting (graft new experiment/eval pairs)"
         type: boolean
         required: false
         default: true
-      commit_to_branch:
-        description: "Commit exported results to the dispatched branch instead of opening a PR"
-        type: boolean
-        required: false
-        default: false
-      cli_stable_version:
+      commit_results:
+        description: "Where to commit exported results JSON"
+        type: choice
+        options:
+          - results-pr
+          - current-branch
+          - none
+        default: results-pr
+      cli_stable_version: &cli_stable_version
         description: "Pin the cli suite's stable CLI version instead of resolving npm's latest dist-tag (blank to resolve)"
+        type: string
         required: false
         default: ""
-      cli_beta_version:
+      cli_beta_version: &cli_beta_version
         description: "Pin the cli suite's beta CLI version instead of resolving npm's beta dist-tag (blank to resolve)"
+        type: string
         required: false
         default: ""
-      cli_next_version:
+      cli_next_version: &cli_next_version
         description: "Pin the cli suite's next CLI version instead of resolving npm's next dist-tag (blank to resolve)"
+        type: string
         required: false
         default: ""
-  schedule:
-    - cron: '15 6 * * *'
+  workflow_call:
+    inputs:
+      experiments: *experiments
+      eval: *eval
+      suite: *suite
+      runs: *runs
+      timeout_sec: *timeout_sec
+      sandbox_concurrency: *sandbox_concurrency
+      merge: *merge
+      cli_stable_version: *cli_stable_version
+      cli_beta_version: *cli_beta_version
+      cli_next_version: *cli_next_version
+      commit_results:
+        type: string
+        default: results-pr
   pull_request:
     # Run whenever a PR carrying the run-evals label is opened, pushed to, or
     # receives the run-evals label. The job-level `if` gates on those cases.
@@ -63,9 +85,9 @@ permissions:
 
 concurrency:
   # The results app's pushes get a unique group so they can't cancel the run that pushed them.
-  group: eval-refresh-${{ github.event.sender.id == 291952723 && github.run_id || github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}
+  group: eval-refresh-${{ github.workflow }}-${{ github.event.sender.id == 291952723 && github.run_id || github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}
   # Supersede an in-flight run when a new commit is pushed to the same PR, but
-  # never cancel a workflow_dispatch run (those open the refresh PR).
+  # never cancel a workflow_dispatch run.
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
@@ -97,10 +119,19 @@ jobs:
       - name: Prepare inputs
         id: inputs
         shell: bash
+        env:
+          COMMIT_RESULTS: ${{ inputs.commit_results }}
         run: |
           set -euo pipefail
 
-          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+          if [ "${{ github.event_name }}" != "pull_request" ]; then
+            case "$COMMIT_RESULTS" in
+              results-pr|current-branch|none) ;;
+              *) echo "::error::Invalid commit_results: $COMMIT_RESULTS"; exit 1 ;;
+            esac
+          fi
+
+          if [ "${{ github.event_name }}" != "pull_request" ]; then
             experiments_override="${{ inputs.experiments }}"
             eval_id="${{ inputs.eval }}"
             suite="${{ inputs.suite }}"
@@ -115,16 +146,6 @@ jobs:
               echo "::error::Set suite or eval to choose which evals to run."
               exit 1
             fi
-          elif [ "${{ github.event_name }}" = "schedule" ]; then
-            experiments_override=""
-            eval_id=""
-            suite="regression,cli"
-            runs="3"
-            timeout_sec="720"
-            sandbox_concurrency="250"
-            cli_stable_version=""
-            cli_beta_version=""
-            cli_next_version=""
           else
             experiments_override=""
             eval_id=""
@@ -516,12 +537,17 @@ jobs:
           echo "PR_URLS=$urls" >> "$GITHUB_ENV"
 
       - name: Upload results to Braintrust
-        # A Braintrust outage shouldn't block publishing results.
-        continue-on-error: true
+        # A Braintrust outage shouldn't block publishing results, but with no results PR it is the only record.
+        continue-on-error: ${{ github.event_name == 'pull_request' || inputs.commit_results != 'none' }}
         env:
           BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
           BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}
-        run: pnpm upload-braintrust --runs "${{ needs.prepare.outputs.runs }}"
+        run: |
+          if [ "${{ inputs.commit_results }}" = "none" ] && { [ -z "$BRAINTRUST_API_KEY" ] || [ -z "$BRAINTRUST_PROJECT_ID" ]; }; then
+            echo "::error::Braintrust credentials are required when commit_results is none."
+            exit 1
+          fi
+          pnpm upload-braintrust --runs "${{ needs.prepare.outputs.runs }}"
 
       - name: Upload exported results
         if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
@@ -533,13 +559,10 @@ jobs:
           retention-days: 7
 
       - name: Commit exported results to branch
-        # PR runs commit to the PR head branch. Manual dispatch commits to the
-        # selected branch only when commit_to_branch is enabled. Scheduled runs
-        # go through a PR instead (see "Create results pull request" below):
-        # main's branch protection rejects a direct push.
+        # PR runs commit to the PR head branch.
         if: >-
           github.event_name == 'pull_request' ||
-          (github.event_name == 'workflow_dispatch' && inputs.commit_to_branch)
+          (github.event_name != 'pull_request' && inputs.commit_results == 'current-branch')
         shell: bash
         run: |
           set -euo pipefail
@@ -563,8 +586,7 @@ jobs:
 
       - name: Create results pull request
         if: >-
-          github.event_name == 'schedule' ||
-          (github.event_name == 'workflow_dispatch' && !inputs.commit_to_branch)
+          github.event_name != 'pull_request' && inputs.commit_results == 'results-pr'
         id: cpr
         uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
         with:

.github/workflows/regression-eval-schedule.yml

@@ -0,0 +1,18 @@
+name: Regression eval schedule
+
+on:
+  schedule:
+    - cron: '15 6 * * *'
+
+permissions:
+  contents: write
+  pull-requests: write
+  actions: read
+
+jobs:
+  run:
+    uses: ./.github/workflows/eval-refresh.yml
+    secrets: inherit
+    with:
+      suite: regression
+      merge: false

CONTRIBUTING.md

@@ -56,7 +56,9 @@ You have a few options to run evals in CI:
 
 - Add the `run-evals-changed` label to your PR to refresh only the `evals/` changed in that PR and commit merged results directly to your branch.
 - Add the `run-evals` label to run every benchmark eval across the experiments in `experiments/benchmark/`. Use this when a change can affect results broadly, such as framework changes.
-- Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow manually to target any branch and choose specific evals, experiments, or other options. It can commit results directly to the selected branch or open a separate results PR.
+- Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow manually to target any branch and choose specific evals, experiments, or other options. Set `commit_results` to `current-branch`, `results-pr`, or `none` to choose where to commit the results JSON.
+
+To schedule a suite, add a team-owned workflow that calls [eval-refresh.yml](.github/workflows/eval-refresh.yml) with the suite and optional experiments, and add the workflow to [CODEOWNERS](.github/CODEOWNERS). See existing schedules in [.github/workflows](.github/workflows/) for reference.
 
 Include refreshed results for PRs with new/changed evals so a reviewer can inspect them in Braintrust.
 
@@ -102,16 +104,21 @@ Common workflows:
 
 ## CLI evals
 
-The CLI team owns `evals/cli/` and its results. CLI evals run on `codex-gpt-6-luna-cli-{pinned,stable,beta,next,nodaemon,absent}` under `experiments/cli/`: `pinned` runs the repo's pinned CLI version, `stable`/`beta`/`next` install the latest stable, beta or next CLI, and `nodaemon`/`absent` additionally force Docker-less sandboxes — comparing the same scenario across CLI environments.
+The CLI team owns `evals/cli/` and its results. CLI evals run on `codex-gpt-6-luna-cli-{pinned,stable,beta,next,nodaemon,absent}` under `experiments/cli/`: `pinned` runs the repo's pinned CLI version, `stable`/`beta`/`next` install the latest stable, beta or next CLI, and `nodaemon`/`absent` additionally force Docker-less sandboxes — comparing the same scenario across CLI environments. The `<model>-cli-{stable,next}` arms run the same scenario on the stable and next CLI across `claude-code-sonnet-5.5`, `claude-code-opus-5.5`, `codex-gpt-6.1-sol` and `opencode-kimi-k3`, comparing models rather than environments.
 
 Which evals each arm picks up:
 
-- **pinned, stable, beta, next** run every `interface: cli` eval that isn't `hostedProject: true`.
+- **pinned, stable, beta, next** run every `interface: cli` eval that isn't `hostedProject: true`. The `<model>-cli-{stable,next}` matrix arms pick up the same evals as `stable` and `next`.
 - **nodaemon, absent** additionally only run evals that also set `needsDocker: false` and `projectRunning: false` — the harness cannot pre-start a stack, or link a hosted project, without Docker.
 
 Common workflows:
 
 - **Add or change a CLI eval.** Add the scenario under `evals/cli/<id>/` (see [Adding an eval](#adding-an-eval)); set `needsDocker: false` in its `PROMPT.md` frontmatter if it can run without Docker, open a PR, and add the `run-evals-changed` label. Results for the changed evals are committed back to your branch and viewable in the Vercel preview.
 - **Refresh every CLI eval.** Dispatch the [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml) workflow on `main` with `suite: cli`. It opens a draft PR with the updated `cli-eval-results.json` for you to review and merge. Leave `cli_stable_version`/`cli_beta_version`/`cli_next_version` blank to resolve npm's latest dist-tags, or pin them to reproduce a specific run.
 - **Analyze results over time.** Every merge that changes `cli-eval-results.json` appends a snapshot to [`cli-results.jsonl`](https://supabase.github.io/evals/cli-results.jsonl) on GitHub Pages, alongside the [benchmark](https://supabase.github.io/evals/results.jsonl), [regression](https://supabase.github.io/evals/regression-results.jsonl), and [docs](https://supabase.github.io/evals/docs-results.jsonl) histories.
+- **Scheduled runs.** The CLI team's schedule calls [Refresh eval results](https://github.com/supabase/evals/actions/workflows/eval-refresh.yml). Results go to Braintrust.
+  - Daily, 04:20 UTC: the six `codex-gpt-6-luna-cli-*` arms.
+  - Weekly, other models on stable and next. See `.github/workflows/cli-eval-schedule.yml` for the schedule and experiment list.
+  - Edit the cadence and experiment lists in `.github/workflows/cli-eval-schedule.yml`.
+  - The shared nightly no longer runs the cli suite.
 - **Run the unit tests.** `pnpm --filter @supabase-evals/framework test:cli-lib` (the CLI skip predicates in `experiments/cli/lib/` plus every CLI eval's scorer tests) — also part of `pnpm test`.

experiments/cli/claude-code-opus-5.5-cli-next.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { claudeCodeOpus55 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...claudeCodeOpus55,
+  localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/claude-code-opus-5.5-cli-stable.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { claudeCodeOpus55 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...claudeCodeOpus55,
+  localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/claude-code-sonnet-5.5-cli-next.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { claudeCodeSonnet55 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...claudeCodeSonnet55,
+  localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/claude-code-sonnet-5.5-cli-stable.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { claudeCodeSonnet55 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...claudeCodeSonnet55,
+  localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/codex-gpt-6.1-sol-cli-next.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { codexGpt61Sol } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...codexGpt61Sol,
+  localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/codex-gpt-6.1-sol-cli-stable.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { codexGpt61Sol } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...codexGpt61Sol,
+  localStack: localStackRuntime({ cliVersion: 'stable', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});

experiments/cli/opencode-kimi-k3-cli-next.experiment.ts

@@ -0,0 +1,11 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { localStackRuntime } from '@supabase-evals/sandbox';
+import { opencodeKimiK3 } from '../presets.js';
+import { EVAL_ENV } from './lib/eval-env.js';
+import { skipUnlessCli } from './lib/skip.js';
+
+export default defineExperiment({
+  ...opencodeKimiK3,
+  localStack: localStackRuntime({ cliVersion: 'next', env: EVAL_ENV }),
+  skipEval: skipUnlessCli,
+});