feat: add Spark REST eval suite

.env.example

@@ -15,3 +15,6 @@ XAI_API_KEY=
 
 BRAINTRUST_PROJECT_ID=
 BRAINTRUST_API_KEY=
+
+# Spark control API used to provision fresh hosted projects.
+SPARK_API_URL=

.github/CODEOWNERS

@@ -9,3 +9,7 @@
 /evals/cli/ @supabase/cli
 /experiments/cli/ @supabase/cli
 /apps/web/src/data/cli-eval-results.json @supabase/cli
+
+/evals/spark/ @supabase/spark
+/experiments/spark/ @supabase/spark
+/skills/spark/ @supabase/spark

README.md

@@ -29,9 +29,9 @@ Agent-backed runs require the relevant provider key in `.env` (e.g. `OPENAI_API_
 
 - An **eval** is one scenario under `evals/<suite>/<id>/`. It contains the prompt, scorer, and optional starting state for the two environments: `remote/` (the hosted project) and `local/` (the agent's working files).
 - An **experiment** is one agent/runtime/model setup under `experiments/<suite>/<name>.experiment.ts`. Its ID remains `<name>`.
-- A **suite** is a named set of evals and the experiments that run them. Each suite gets a folder under `evals/` and `experiments/`. Experiments used in several suites can use a shared preset from `experiments/presets.ts`. The suites are `benchmark`, `regression`, `docs`, and `cli`.
+- A **suite** is a named set of evals and the experiments that run them. Each suite gets a folder under `evals/` and `experiments/`. Experiments used in several suites can use a shared preset from `experiments/presets.ts`. The suites are defined in `packages/core/src/eval-metadata.ts`.
 - An **agent** is the model driver that receives the eval prompt and calls the configured tools.
-- A **runtime** is the local Supabase-like environment and tool surface an experiment gives to the agent.
+- A **runtime** is the local or hosted environment and tool surface an experiment gives to the agent.
 - `platform-lite` exposes a Supabase Management API-compatible HTTP surface backed by [`@supabase/lite`](https://github.com/supabase/supabase-lite), so real tools like `@supabase/mcp-server-supabase` can run against a lightweight project.
 
 ## Running evals
@@ -119,7 +119,7 @@ There are two runtimes, chosen automatically per eval:
 - **Tools evals** run the agent against the experiment's MCP/tool surface (no `local/` directory, no `interface: cli`), then score the resulting project state or report.
 - **Local-stack evals** run the agent inside a Docker sandbox — a `bash` tool plus file tools with the real Supabase CLI installed — so it can run `supabase init/start/db/test` against a real local stack. An eval uses this runtime when it ships a `local/` workspace **or** declares `interface: cli` (the latter covers bootstrap scenarios that start from an empty workspace).
 
-`interface` (`mcp` | `cli`) is otherwise a benchmark dimension (a cross-team KPI label), not the runtime switch — the `local/` directory and `interface: cli` are what decide whether a sandbox boots.
+`interface` (`mcp` | `cli` | `rest`) is otherwise a benchmark dimension (a cross-team KPI label), not the runtime switch — the `local/` directory and `interface: cli` are what decide whether a sandbox boots.
 
 ### Local-stack evals
 

apps/framework/harness/run-eval.ts

@@ -520,22 +520,21 @@ async function runOne(
     };
   }
 
-  // Tools mode: a CLI agent gets a bare sandbox reaching platform-lite via
-  // host.docker.internal. An in-process agent runs host-side with no sandbox.
+  await using session = disposable(
+    await exp.runtime.startSession({
+      ...readSessionSeedArgs(ev),
+      hostname: agentRunsInSandbox ? '0.0.0.0' : undefined,
+    })
+  );
   await using cliSandbox = agentRunsInSandbox
     ? disposable(
         await createBareSandbox({
+          env: session.env,
           skills: skillSources,
           mounts: supabaseMcpServerMounts(),
         })
       )
     : undefined;
-  await using session = disposable(
-    await exp.runtime.startSession({
-      ...readSessionSeedArgs(ev),
-      hostname: agentRunsInSandbox ? '0.0.0.0' : undefined,
-    })
-  );
 
   // A CLI agent discovers its installed skills itself. An in-process agent has
   // no filesystem, so its skills are advertised in the prompt and pulled on

apps/framework/package.json

@@ -10,10 +10,11 @@
     "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke",
     "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts",
     "typecheck": "tsc --noEmit",
-    "test": "vitest run && pnpm test:framework && pnpm test:cli-lib",
+    "test": "vitest run && pnpm test:framework && pnpm test:cli-lib && pnpm test:spark",
     "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
     "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts scripts/export-results.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts",
     "test:cli-lib": "vitest run --config vitest.cli-lib.config.ts --passWithNoTests experiments/cli evals/cli",
+    "test:spark": "vitest run --config vitest.cli-lib.config.ts experiments/spark evals/spark",
     "export-results": "node --import tsx/esm scripts/export-results.ts",
     "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts",
     "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts",

apps/framework/vitest.cli-lib.config.ts

@@ -1,3 +1,3 @@
 import { defineConfig } from 'vitest/config';
 
-export default defineConfig({ test: { name: 'cli evals', root: '../..' } });
+export default defineConfig({ test: { name: 'evals', root: '../..' } });

evals/spark/build-database-001-task-due-dates/EVAL.test.ts

@@ -0,0 +1,152 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { sparkRuntime } from '../../../experiments/spark/lib/runtime.js';
+import scorer from './EVAL.js';
+
+const tasks = [
+  'Write release notes',
+  'Ship task tracker',
+  'Review schema',
+  'Plan next iteration',
+].map((title, index) => ({
+  id: index + 1,
+  title,
+  completed: index === 2,
+  created_at: `2026-11-01 12:0${index}:00`,
+  due_date: index === 1 ? '2026-11-10' : null,
+}));
+
+afterEach(() => vi.unstubAllGlobals());
+
+async function score(
+  options: {
+    columns?: Record<string, unknown>[];
+    tasks?: unknown;
+    tableStatus?: number;
+    reportPasses?: boolean;
+  } = {}
+) {
+  vi.stubGlobal('fetch', async (input: string | URL | Request) => {
+    const url = input instanceof Request ? new URL(input.url) : new URL(input);
+    if (url.pathname === '/v1/projects')
+      return Response.json({
+        ref: 'project',
+        url: 'https://project.example',
+        api_keys: [
+          { type: 'secret', api_key: 'secret' },
+          { type: 'publishable', api_key: 'publishable' },
+        ],
+      });
+    if (url.pathname === '/database/query')
+      return Response.json(
+        options.columns ?? [{ name: 'due_date', type: 'date', not_null: 0 }]
+      );
+    expect(url.pathname).toBe('/rest/v1/tasks');
+    return options.tableStatus
+      ? Response.json(
+          { message: 'table unavailable' },
+          { status: options.tableStatus }
+        )
+      : Response.json(options.tasks ?? tasks);
+  });
+  const session = await sparkRuntime('https://control.example').startSession(
+    {}
+  );
+  const judge = vi.fn(async () => ({ passed: options.reportPasses ?? true }));
+  try {
+    const result = await scorer({
+      ...session.scoringContext,
+      toolCalls: [],
+      transcript: [],
+      judge,
+    });
+    return { result, judge };
+  } finally {
+    await session.close();
+  }
+}
+
+describe('task state checks', () => {
+  it('rejects missing dates, altered seed data and deleted rows', async () => {
+    for (const [changed, name] of [
+      [
+        tasks.map((task) => ({ ...task, due_date: null })),
+        'Requested due date saved and other tasks undated',
+      ],
+      [
+        tasks.map((task) => ({ ...task, title: 'Replaced' })),
+        'Existing task data preserved',
+      ],
+      [
+        tasks.map((task) => ({ ...task, created_at: '2026-11-02T00:00:00' })),
+        'Existing task data preserved',
+      ],
+      [
+        tasks.map((task) => ({ ...task, completed: 'false' })),
+        'Existing task data preserved',
+      ],
+      [tasks.slice(1), 'Existing task data preserved'],
+      [[], 'Existing task data preserved'],
+    ])
+      expect((await score({ tasks: changed })).result.checks).toContainEqual(
+        expect.objectContaining({ name, passed: false })
+      );
+  });
+});
+
+describe('hosted task scorer', () => {
+  it('accepts the complete result and rejects an incorrect report', async () => {
+    expect((await score()).result.passed).toBe(true);
+    expect((await score({ reportPasses: false })).result.passed).toBe(false);
+  });
+
+  it('accepts a matching date column after an unrelated date column', async () => {
+    const renamed = tasks.map(({ due_date, ...task }) => ({
+      ...task,
+      deadline: due_date,
+    }));
+    expect(
+      (
+        await score({
+          columns: [
+            { name: 'another_date', type: 'date', not_null: 0 },
+            { name: 'deadline', type: 'date', not_null: 0 },
+          ],
+          tasks: renamed,
+        })
+      ).result.passed
+    ).toBe(true);
+  });
+
+  it('rejects missing, non-date and required columns', async () => {
+    for (const columns of [
+      [],
+      [{ name: 'due_date', type: 'text', not_null: 0 }],
+      [{ name: 'due_date', type: 'date', not_null: 1 }],
+    ]) {
+      const { result } = await score({ columns });
+      expect(result.passed).toBe(false);
+      expect(
+        result.checks?.find(
+          (check) => check.name === 'Nullable date column added'
+        )?.passed
+      ).toBe(false);
+    }
+  });
+
+  it('records unreadable state and still judges the report', async () => {
+    const { result, judge } = await score({ tableStatus: 404 });
+    expect(result.passed).toBe(false);
+    expect(result.checks?.map((check) => check.name)).toEqual([
+      'Nullable date column added',
+      'Existing task data preserved',
+      'Requested due date saved and other tasks undated',
+      'Open tasks reported in due-date order',
+    ]);
+    expect(result.checks?.slice(0, 3)).toEqual(
+      Array(3).fill(
+        expect.objectContaining({ passed: false, notes: expect.any(String) })
+      )
+    );
+    expect(judge).toHaveBeenCalledOnce();
+  });
+});

evals/spark/build-database-001-task-due-dates/EVAL.ts

@@ -0,0 +1,102 @@
+import type {
+  CheckResult,
+  ToolScorer,
+  ToolScoringContext,
+} from '@supabase-evals/core';
+import { z } from 'zod';
+
+const tasksSchema = z.array(
+  z
+    .object({
+      id: z.number(),
+      title: z.string(),
+      completed: z.boolean(),
+      created_at: z.string(),
+    })
+    .catchall(z.unknown())
+);
+
+async function readTaskState(ctx: ToolScoringContext) {
+  try {
+    const { rows: addedColumns } = await ctx.query(
+      `select name, type, "notnull" as not_null from pragma_table_info('tasks') where name not in ('id', 'title', 'completed', 'created_at')`
+    );
+    const { data: rawTasks, error: taskReadError } = await ctx.client
+      .from('tasks')
+      .select('*')
+      .order('id');
+    if (taskReadError) throw new Error(taskReadError.message);
+    const tasks = tasksSchema.parse(rawTasks);
+    const dateColumns = addedColumns.flatMap((column) =>
+      typeof column.name === 'string' &&
+      typeof column.type === 'string' &&
+      column.type.toLowerCase() === 'date' &&
+      column.not_null === 0
+        ? [column.name]
+        : []
+    );
+    return { tasks, dateColumns, error: undefined };
+  } catch (error) {
+    return {
+      tasks: [],
+      dateColumns: [],
+      error: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+const scorer: ToolScorer = async (ctx) => {
+  const { tasks, dateColumns, error } = await readTaskState(ctx);
+  const titles = [
+    'Write release notes',
+    'Ship task tracker',
+    'Review schema',
+    'Plan next iteration',
+  ];
+  const createdAt = [
+    '2026-11-01 12:00:00',
+    '2026-11-01 12:01:00',
+    '2026-11-01 12:02:00',
+    '2026-11-01 12:03:00',
+  ];
+  const report = await ctx.judge({
+    input: ctx.agentReport ?? '',
+    rubric:
+      'The response must list exactly these open tasks with their due dates: Ship task tracker, November 10, 2026; Write release notes, no due date; Plan next iteration, no due date. Ship task tracker must appear before the undated tasks. Either order among the undated tasks is valid. Review schema is complete and must not appear as an open task. Task IDs are optional. Accept equivalent date formatting and descriptions of an absent date.',
+  });
+  const checks: CheckResult[] = [
+    {
+      name: 'Nullable date column added',
+      passed: dateColumns.length > 0,
+      notes: error,
+    },
+    {
+      name: 'Existing task data preserved',
+      passed:
+        tasks.length === titles.length &&
+        tasks.every(
+          (task, index) =>
+            task.id === index + 1 &&
+            task.title === titles[index] &&
+            task.completed === (task.id === 3) &&
+            task.created_at === createdAt[index]
+        ),
+      notes: error,
+    },
+    {
+      name: 'Requested due date saved and other tasks undated',
+      passed:
+        tasks.length === titles.length &&
+        dateColumns.some((column) =>
+          tasks.every(
+            (task) => task[column] === (task.id === 2 ? '2026-11-10' : null)
+          )
+        ),
+      notes: error,
+    },
+    { name: 'Open tasks reported in due-date order', ...report },
+  ];
+  return { passed: checks.every((check) => check.passed), checks };
+};
+
+export default scorer;

evals/spark/build-database-001-task-due-dates/PROMPT.md

@@ -0,0 +1,12 @@
+---
+stage: build
+interface: rest
+product:
+  - database
+  - data-api
+topic:
+  - migrations
+motivation: https://linear.app/supabase/issue/AI-1399/add-an-agent-eval-suite-for-spark-rest-api
+---
+
+Can you add optional due dates to our task tracker in the provided Spark project? Set "Ship task tracker" to November 10, 2026 and leave the other tasks undated. Then show me the open tasks with their due dates, soonest first and undated tasks last.

evals/spark/build-database-001-task-due-dates/remote/project.sql

@@ -0,0 +1,12 @@
+create table public.tasks (
+  id integer primary key,
+  title text not null,
+  completed boolean not null default false,
+  created_at timestamptz not null default now()
+);
+
+insert into public.tasks (id, title, completed, created_at) values
+  (1, 'Write release notes', false, '2026-11-01T12:00:00Z'),
+  (2, 'Ship task tracker', false, '2026-11-01T12:01:00Z'),
+  (3, 'Review schema', true, '2026-11-01T12:02:00Z'),
+  (4, 'Plan next iteration', false, '2026-11-01T12:03:00Z');

experiments/spark/codex-gpt-6-luna.experiment.ts

@@ -0,0 +1,9 @@
+import { defineExperiment } from '@supabase-evals/core';
+import { codexGpt6Luna } from '../presets.js';
+import { sparkRuntime } from './lib/runtime.js';
+
+export default defineExperiment({
+  agent: codexGpt6Luna.agent,
+  runtime: sparkRuntime(process.env.SPARK_API_URL ?? ''),
+  skills: [...codexGpt6Luna.skills, 'spark'],
+});