diff --git a/.env.example b/.env.example index a076d26f5..04fa7979c 100644 --- a/.env.example +++ b/.env.example @@ -15,3 +15,6 @@ XAI_API_KEY= BRAINTRUST_PROJECT_ID= BRAINTRUST_API_KEY= + +# Spark control API used to provision fresh hosted projects. +SPARK_API_URL= diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f63d16b19..19194a394 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,3 +9,7 @@ /evals/cli/ @supabase/cli /experiments/cli/ @supabase/cli /apps/web/src/data/cli-eval-results.json @supabase/cli + +/evals/spark/ @supabase/spark +/experiments/spark/ @supabase/spark +/skills/spark/ @supabase/spark diff --git a/README.md b/README.md index ce0bb0644..b859d7ee7 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,9 @@ Agent-backed runs require the relevant provider key in `.env` (e.g. `OPENAI_API_ - An **eval** is one scenario under `evals///`. It contains the prompt, scorer, and optional starting state for the two environments: `remote/` (the hosted project) and `local/` (the agent's working files). - An **experiment** is one agent/runtime/model setup under `experiments//.experiment.ts`. Its ID remains ``. -- A **suite** is a named set of evals and the experiments that run them. Each suite gets a folder under `evals/` and `experiments/`. Experiments used in several suites can use a shared preset from `experiments/presets.ts`. The suites are `benchmark`, `regression`, `docs`, and `cli`. +- A **suite** is a named set of evals and the experiments that run them. Each suite gets a folder under `evals/` and `experiments/`. Experiments used in several suites can use a shared preset from `experiments/presets.ts`. The suites are defined in `packages/core/src/eval-metadata.ts`. - An **agent** is the model driver that receives the eval prompt and calls the configured tools. -- A **runtime** is the local Supabase-like environment and tool surface an experiment gives to the agent. +- A **runtime** is the local or hosted environment and tool surface an experiment gives to the agent. - `platform-lite` exposes a Supabase Management API-compatible HTTP surface backed by [`@supabase/lite`](https://github.com/supabase/supabase-lite), so real tools like `@supabase/mcp-server-supabase` can run against a lightweight project. ## Running evals @@ -119,7 +119,7 @@ There are two runtimes, chosen automatically per eval: - **Tools evals** run the agent against the experiment's MCP/tool surface (no `local/` directory, no `interface: cli`), then score the resulting project state or report. - **Local-stack evals** run the agent inside a Docker sandbox — a `bash` tool plus file tools with the real Supabase CLI installed — so it can run `supabase init/start/db/test` against a real local stack. An eval uses this runtime when it ships a `local/` workspace **or** declares `interface: cli` (the latter covers bootstrap scenarios that start from an empty workspace). -`interface` (`mcp` | `cli`) is otherwise a benchmark dimension (a cross-team KPI label), not the runtime switch — the `local/` directory and `interface: cli` are what decide whether a sandbox boots. +`interface` (`mcp` | `cli` | `rest`) is otherwise a benchmark dimension (a cross-team KPI label), not the runtime switch — the `local/` directory and `interface: cli` are what decide whether a sandbox boots. ### Local-stack evals diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 2eeb145f7..a64926d76 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -520,22 +520,21 @@ async function runOne( }; } - // Tools mode: a CLI agent gets a bare sandbox reaching platform-lite via - // host.docker.internal. An in-process agent runs host-side with no sandbox. + await using session = disposable( + await exp.runtime.startSession({ + ...readSessionSeedArgs(ev), + hostname: agentRunsInSandbox ? '0.0.0.0' : undefined, + }) + ); await using cliSandbox = agentRunsInSandbox ? disposable( await createBareSandbox({ + env: session.env, skills: skillSources, mounts: supabaseMcpServerMounts(), }) ) : undefined; - await using session = disposable( - await exp.runtime.startSession({ - ...readSessionSeedArgs(ev), - hostname: agentRunsInSandbox ? '0.0.0.0' : undefined, - }) - ); // A CLI agent discovers its installed skills itself. An in-process agent has // no filesystem, so its skills are advertised in the prompt and pulled on diff --git a/apps/framework/package.json b/apps/framework/package.json index 2b667f3ac..df349fbb4 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -10,10 +10,11 @@ "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts", "typecheck": "tsc --noEmit", - "test": "vitest run && pnpm test:framework && pnpm test:cli-lib", + "test": "vitest run && pnpm test:framework && pnpm test:cli-lib && pnpm test:spark", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts scripts/export-results.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts", "test:cli-lib": "vitest run --config vitest.cli-lib.config.ts --passWithNoTests experiments/cli evals/cli", + "test:spark": "vitest run --config vitest.cli-lib.config.ts experiments/spark evals/spark", "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts", diff --git a/apps/framework/vitest.cli-lib.config.ts b/apps/framework/vitest.cli-lib.config.ts index 571d03546..f009d83a7 100644 --- a/apps/framework/vitest.cli-lib.config.ts +++ b/apps/framework/vitest.cli-lib.config.ts @@ -1,3 +1,3 @@ import { defineConfig } from 'vitest/config'; -export default defineConfig({ test: { name: 'cli evals', root: '../..' } }); +export default defineConfig({ test: { name: 'evals', root: '../..' } }); diff --git a/evals/spark/build-database-001-task-due-dates/EVAL.test.ts b/evals/spark/build-database-001-task-due-dates/EVAL.test.ts new file mode 100644 index 000000000..5a5464f5e --- /dev/null +++ b/evals/spark/build-database-001-task-due-dates/EVAL.test.ts @@ -0,0 +1,152 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { sparkRuntime } from '../../../experiments/spark/lib/runtime.js'; +import scorer from './EVAL.js'; + +const tasks = [ + 'Write release notes', + 'Ship task tracker', + 'Review schema', + 'Plan next iteration', +].map((title, index) => ({ + id: index + 1, + title, + completed: index === 2, + created_at: `2026-11-01 12:0${index}:00`, + due_date: index === 1 ? '2026-11-10' : null, +})); + +afterEach(() => vi.unstubAllGlobals()); + +async function score( + options: { + columns?: Record[]; + tasks?: unknown; + tableStatus?: number; + reportPasses?: boolean; + } = {} +) { + vi.stubGlobal('fetch', async (input: string | URL | Request) => { + const url = input instanceof Request ? new URL(input.url) : new URL(input); + if (url.pathname === '/v1/projects') + return Response.json({ + ref: 'project', + url: 'https://project.example', + api_keys: [ + { type: 'secret', api_key: 'secret' }, + { type: 'publishable', api_key: 'publishable' }, + ], + }); + if (url.pathname === '/database/query') + return Response.json( + options.columns ?? [{ name: 'due_date', type: 'date', not_null: 0 }] + ); + expect(url.pathname).toBe('/rest/v1/tasks'); + return options.tableStatus + ? Response.json( + { message: 'table unavailable' }, + { status: options.tableStatus } + ) + : Response.json(options.tasks ?? tasks); + }); + const session = await sparkRuntime('https://control.example').startSession( + {} + ); + const judge = vi.fn(async () => ({ passed: options.reportPasses ?? true })); + try { + const result = await scorer({ + ...session.scoringContext, + toolCalls: [], + transcript: [], + judge, + }); + return { result, judge }; + } finally { + await session.close(); + } +} + +describe('task state checks', () => { + it('rejects missing dates, altered seed data and deleted rows', async () => { + for (const [changed, name] of [ + [ + tasks.map((task) => ({ ...task, due_date: null })), + 'Requested due date saved and other tasks undated', + ], + [ + tasks.map((task) => ({ ...task, title: 'Replaced' })), + 'Existing task data preserved', + ], + [ + tasks.map((task) => ({ ...task, created_at: '2026-11-02T00:00:00' })), + 'Existing task data preserved', + ], + [ + tasks.map((task) => ({ ...task, completed: 'false' })), + 'Existing task data preserved', + ], + [tasks.slice(1), 'Existing task data preserved'], + [[], 'Existing task data preserved'], + ]) + expect((await score({ tasks: changed })).result.checks).toContainEqual( + expect.objectContaining({ name, passed: false }) + ); + }); +}); + +describe('hosted task scorer', () => { + it('accepts the complete result and rejects an incorrect report', async () => { + expect((await score()).result.passed).toBe(true); + expect((await score({ reportPasses: false })).result.passed).toBe(false); + }); + + it('accepts a matching date column after an unrelated date column', async () => { + const renamed = tasks.map(({ due_date, ...task }) => ({ + ...task, + deadline: due_date, + })); + expect( + ( + await score({ + columns: [ + { name: 'another_date', type: 'date', not_null: 0 }, + { name: 'deadline', type: 'date', not_null: 0 }, + ], + tasks: renamed, + }) + ).result.passed + ).toBe(true); + }); + + it('rejects missing, non-date and required columns', async () => { + for (const columns of [ + [], + [{ name: 'due_date', type: 'text', not_null: 0 }], + [{ name: 'due_date', type: 'date', not_null: 1 }], + ]) { + const { result } = await score({ columns }); + expect(result.passed).toBe(false); + expect( + result.checks?.find( + (check) => check.name === 'Nullable date column added' + )?.passed + ).toBe(false); + } + }); + + it('records unreadable state and still judges the report', async () => { + const { result, judge } = await score({ tableStatus: 404 }); + expect(result.passed).toBe(false); + expect(result.checks?.map((check) => check.name)).toEqual([ + 'Nullable date column added', + 'Existing task data preserved', + 'Requested due date saved and other tasks undated', + 'Open tasks reported in due-date order', + ]); + expect(result.checks?.slice(0, 3)).toEqual( + Array(3).fill( + expect.objectContaining({ passed: false, notes: expect.any(String) }) + ) + ); + expect(judge).toHaveBeenCalledOnce(); + }); +}); diff --git a/evals/spark/build-database-001-task-due-dates/EVAL.ts b/evals/spark/build-database-001-task-due-dates/EVAL.ts new file mode 100644 index 000000000..f7cc4b32b --- /dev/null +++ b/evals/spark/build-database-001-task-due-dates/EVAL.ts @@ -0,0 +1,102 @@ +import type { + CheckResult, + ToolScorer, + ToolScoringContext, +} from '@supabase-evals/core'; +import { z } from 'zod'; + +const tasksSchema = z.array( + z + .object({ + id: z.number(), + title: z.string(), + completed: z.boolean(), + created_at: z.string(), + }) + .catchall(z.unknown()) +); + +async function readTaskState(ctx: ToolScoringContext) { + try { + const { rows: addedColumns } = await ctx.query( + `select name, type, "notnull" as not_null from pragma_table_info('tasks') where name not in ('id', 'title', 'completed', 'created_at')` + ); + const { data: rawTasks, error: taskReadError } = await ctx.client + .from('tasks') + .select('*') + .order('id'); + if (taskReadError) throw new Error(taskReadError.message); + const tasks = tasksSchema.parse(rawTasks); + const dateColumns = addedColumns.flatMap((column) => + typeof column.name === 'string' && + typeof column.type === 'string' && + column.type.toLowerCase() === 'date' && + column.not_null === 0 + ? [column.name] + : [] + ); + return { tasks, dateColumns, error: undefined }; + } catch (error) { + return { + tasks: [], + dateColumns: [], + error: error instanceof Error ? error.message : String(error), + }; + } +} + +const scorer: ToolScorer = async (ctx) => { + const { tasks, dateColumns, error } = await readTaskState(ctx); + const titles = [ + 'Write release notes', + 'Ship task tracker', + 'Review schema', + 'Plan next iteration', + ]; + const createdAt = [ + '2026-11-01 12:00:00', + '2026-11-01 12:01:00', + '2026-11-01 12:02:00', + '2026-11-01 12:03:00', + ]; + const report = await ctx.judge({ + input: ctx.agentReport ?? '', + rubric: + 'The response must list exactly these open tasks with their due dates: Ship task tracker, November 10, 2026; Write release notes, no due date; Plan next iteration, no due date. Ship task tracker must appear before the undated tasks. Either order among the undated tasks is valid. Review schema is complete and must not appear as an open task. Task IDs are optional. Accept equivalent date formatting and descriptions of an absent date.', + }); + const checks: CheckResult[] = [ + { + name: 'Nullable date column added', + passed: dateColumns.length > 0, + notes: error, + }, + { + name: 'Existing task data preserved', + passed: + tasks.length === titles.length && + tasks.every( + (task, index) => + task.id === index + 1 && + task.title === titles[index] && + task.completed === (task.id === 3) && + task.created_at === createdAt[index] + ), + notes: error, + }, + { + name: 'Requested due date saved and other tasks undated', + passed: + tasks.length === titles.length && + dateColumns.some((column) => + tasks.every( + (task) => task[column] === (task.id === 2 ? '2026-11-10' : null) + ) + ), + notes: error, + }, + { name: 'Open tasks reported in due-date order', ...report }, + ]; + return { passed: checks.every((check) => check.passed), checks }; +}; + +export default scorer; diff --git a/evals/spark/build-database-001-task-due-dates/PROMPT.md b/evals/spark/build-database-001-task-due-dates/PROMPT.md new file mode 100644 index 000000000..99608fc35 --- /dev/null +++ b/evals/spark/build-database-001-task-due-dates/PROMPT.md @@ -0,0 +1,12 @@ +--- +stage: build +interface: rest +product: + - database + - data-api +topic: + - migrations +motivation: https://linear.app/supabase/issue/AI-1399/add-an-agent-eval-suite-for-spark-rest-api +--- + +Can you add optional due dates to our task tracker in the provided Spark project? Set "Ship task tracker" to November 10, 2026 and leave the other tasks undated. Then show me the open tasks with their due dates, soonest first and undated tasks last. diff --git a/evals/spark/build-database-001-task-due-dates/remote/project.sql b/evals/spark/build-database-001-task-due-dates/remote/project.sql new file mode 100644 index 000000000..43218843d --- /dev/null +++ b/evals/spark/build-database-001-task-due-dates/remote/project.sql @@ -0,0 +1,12 @@ +create table public.tasks ( + id integer primary key, + title text not null, + completed boolean not null default false, + created_at timestamptz not null default now() +); + +insert into public.tasks (id, title, completed, created_at) values + (1, 'Write release notes', false, '2026-11-01T12:00:00Z'), + (2, 'Ship task tracker', false, '2026-11-01T12:01:00Z'), + (3, 'Review schema', true, '2026-11-01T12:02:00Z'), + (4, 'Plan next iteration', false, '2026-11-01T12:03:00Z'); diff --git a/experiments/spark/codex-gpt-6-luna.experiment.ts b/experiments/spark/codex-gpt-6-luna.experiment.ts new file mode 100644 index 000000000..3e5c543d3 --- /dev/null +++ b/experiments/spark/codex-gpt-6-luna.experiment.ts @@ -0,0 +1,9 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { codexGpt6Luna } from '../presets.js'; +import { sparkRuntime } from './lib/runtime.js'; + +export default defineExperiment({ + agent: codexGpt6Luna.agent, + runtime: sparkRuntime(process.env.SPARK_API_URL ?? ''), + skills: [...codexGpt6Luna.skills, 'spark'], +}); diff --git a/experiments/spark/lib/runtime.test.ts b/experiments/spark/lib/runtime.test.ts new file mode 100644 index 000000000..d062dc9d1 --- /dev/null +++ b/experiments/spark/lib/runtime.test.ts @@ -0,0 +1,123 @@ +import { mkdtemp, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, expect, it, vi } from 'vitest'; +import { sparkRuntime } from './runtime.js'; + +afterEach(() => vi.unstubAllGlobals()); + +it('creates fresh projects, seeds before execution and reads scoring state', async () => { + const dir = await mkdtemp(join(tmpdir(), 'spark-runtime-')); + const fixture = join(dir, 'project.sql'); + await writeFile(fixture, 'create table tasks (id integer);'); + let projectCount = 0; + const fetchMock = vi.fn(async (url: URL, options?: RequestInit) => { + if (url.pathname === '/v1/projects') { + projectCount++; + return Response.json({ + ref: `project${projectCount}`, + url: `https://project${projectCount}.example`, + api_keys: [ + { type: 'secret', api_key: 'secret' }, + { type: 'publishable', api_key: 'publishable' }, + ], + }); + } + expect(options?.headers).toEqual({ + apikey: 'secret', + 'content-type': 'application/json', + }); + if (url.pathname === '/system/v1/migrations') { + expect(JSON.parse(String(options?.body))).toEqual({ + version: '1', + name: 'eval_fixture', + sql: 'create table tasks (id integer);', + }); + return Response.json({}); + } + expect(url.pathname).toBe('/database/query'); + expect(JSON.parse(String(options?.body))).toEqual({ + query: 'select 1', + }); + return Response.json([{ value: 1 }]); + }); + vi.stubGlobal('fetch', fetchMock); + try { + const runtime = sparkRuntime('https://control.example'); + const session = await runtime.startSession({ projectSeedSql: fixture }); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(session.env?.SPARK_PROJECT_URL).toBe('https://project1.example'); + expect(session.mcpServers).toEqual({}); + expect(await session.scoringContext.query('select 1')).toEqual({ + rows: [{ value: 1 }], + }); + const next = await runtime.startSession({}); + expect(next.scoringContext.ref).not.toBe(session.scoringContext.ref); + await session.close(); + await next.close(); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +it('rejects configuration and unsupported fixtures before provisioning', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + await expect(sparkRuntime('').startSession({})).rejects.toThrow( + 'SPARK_API_URL' + ); + await expect( + sparkRuntime('https://control.example').startSession({ + logsSeedJsonl: 'logs.jsonl', + }) + ).rejects.toThrow('project.sql'); + await expect( + sparkRuntime('https://control.example').startSession({ + projectSeedSql: '/missing-fixture.sql', + }) + ).rejects.toThrow(); + expect(fetchMock).not.toHaveBeenCalled(); +}); + +it('rejects failed provisioning and missing project keys', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response('', { status: 503 })) + .mockResolvedValueOnce( + Response.json({ ref: 'p', url: 'https://p.example', api_keys: [] }) + ); + vi.stubGlobal('fetch', fetchMock); + const runtime = sparkRuntime('https://control.example'); + await expect(runtime.startSession({})).rejects.toThrow('503'); + await expect(runtime.startSession({})).rejects.toThrow('missing API keys'); +}); + +it('stops setup when Spark rejects the fixture', async () => { + const dir = await mkdtemp(join(tmpdir(), 'spark-fixture-')); + const fixture = join(dir, 'project.sql'); + await writeFile(fixture, 'invalid SQL'); + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + Response.json({ + ref: 'project', + url: 'https://project.example', + api_keys: [ + { type: 'secret', api_key: 'secret' }, + { type: 'publishable', api_key: 'publishable' }, + ], + }) + ) + .mockResolvedValueOnce(new Response('fixture rejected', { status: 400 })); + vi.stubGlobal('fetch', fetchMock); + try { + await expect( + sparkRuntime('https://control.example').startSession({ + projectSeedSql: fixture, + }) + ).rejects.toThrow('Spark fixture failed (400): fixture rejected'); + expect(fetchMock).toHaveBeenCalledTimes(2); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/experiments/spark/lib/runtime.ts b/experiments/spark/lib/runtime.ts new file mode 100644 index 000000000..239c86872 --- /dev/null +++ b/experiments/spark/lib/runtime.ts @@ -0,0 +1,115 @@ +import { readFile } from 'node:fs/promises'; +import { createClient } from '@supabase/supabase-js'; +import { createManagementApiClient } from '@supabase-evals/platform-lite'; +import { z } from 'zod'; +import type { EvalRuntime } from '@supabase-evals/core'; + +const projectSchema = z.object({ + ref: z.string().min(1), + url: z.url(), + api_keys: z.array(z.object({ type: z.string(), api_key: z.string().min(1) })), +}); +const rowsSchema = z.array(z.record(z.string(), z.unknown())); + +export function sparkRuntime(apiUrl: string): EvalRuntime { + return { + id: 'spark', + async startSession(args) { + if (!apiUrl.trim()) + throw new Error('Set SPARK_API_URL to the Spark control API URL'); + + if (args.logsSeedJsonl || args.functionsSeedDir || args.pgvector) + throw new Error( + 'Spark REST runtime only supports project.sql fixtures' + ); + + const seedSql = args.projectSeedSql + ? await readFile(args.projectSeedSql, 'utf8') + : undefined; + const createResponse = await fetch(new URL('/v1/projects', apiUrl), { + method: 'POST', + signal: AbortSignal.timeout(30_000), + }); + if (!createResponse.ok) + throw new Error( + `Spark project creation failed (${createResponse.status})` + ); + const project = projectSchema.parse(await createResponse.json()); + + const secret = project.api_keys.find( + (key) => key.type === 'secret' + )?.api_key; + const publishable = project.api_keys.find( + (key) => key.type === 'publishable' + )?.api_key; + if (!secret || !publishable) + throw new Error(`Spark project ${project.ref} is missing API keys`); + + // shortcut: projects remain allocated, add teardown once a deletion endpoint is confirmed. + process.stderr.write(`Spark project created: ${project.ref}\n`); + + const headers = { apikey: secret, 'content-type': 'application/json' }; + if (seedSql) { + const seedResponse = await fetch( + new URL('/system/v1/migrations', project.url), + { + method: 'POST', + headers, + body: JSON.stringify({ + version: '1', + name: 'eval_fixture', + sql: seedSql, + }), + signal: AbortSignal.timeout(30_000), + } + ); + if (!seedResponse.ok) + throw new Error( + `Spark fixture failed (${seedResponse.status}): ${await seedResponse.text()}` + ); + } + + const getClient = () => + createClient(project.url, secret, { + auth: { persistSession: false, autoRefreshToken: false }, + }); + return { + mcpServers: {}, + env: { + SPARK_PROJECT_URL: project.url, + SPARK_SECRET_KEY: secret, + SPARK_PUBLISHABLE_KEY: publishable, + }, + scoringContext: { + ref: project.ref, + // Required by ToolScoringContext, but Spark's Management API is incomplete. + mgmt: createManagementApiClient(apiUrl, secret), + client: getClient(), + getClient, + async query(sql) { + const response = await fetch( + new URL('/database/query', project.url), + { + method: 'POST', + headers, + body: JSON.stringify({ query: sql }), + signal: AbortSignal.timeout(30_000), + } + ); + if (!response.ok) + throw new Error( + `Spark scoring query failed (${response.status}): ${await response.text()}` + ); + return { rows: rowsSchema.parse(await response.json()) }; + }, + async invokeFunction() { + throw new Error( + 'Spark REST runtime does not support function invocation' + ); + }, + }, + async close() {}, + }; + }, + }; +} diff --git a/package.json b/package.json index afcc38414..fb5834507 100644 --- a/package.json +++ b/package.json @@ -24,9 +24,12 @@ "dependencies": { "@ai-sdk/anthropic": "catalog:", "@ai-sdk/openai": "catalog:", + "@supabase-evals/platform-lite": "workspace:*", + "@supabase/supabase-js": "catalog:", "common-tags": "^1.8.2", "shell-quote": "^1.11.0", - "smol-toml": "^1.9.1" + "smol-toml": "^1.9.1", + "zod": "catalog:" }, "devDependencies": { "@biomejs/biome": "1.9.4", @@ -35,7 +38,8 @@ "@types/common-tags": "^1.8.4", "@types/node": "catalog:", "@types/shell-quote": "^1.7.5", - "typescript": "catalog:" + "typescript": "catalog:", + "vitest": "catalog:" }, "engines": { "node": "24.x", diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 2a678a399..0762689f1 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -46,6 +46,7 @@ export const evalSuiteSchema = z.enum([ 'docs', 'cli', 'docs-nav', + 'spark', ]); export const EVAL_SUITES = evalSuiteSchema.options; export type EvalSuite = z.infer; @@ -90,6 +91,7 @@ export type ExperimentDisplayMetadata = z.infer< * Interface(s) the agent uses to act on Supabase — a benchmark dimension * (cross-team KPI), not the runtime switch. `mcp` = the platform-lite MCP/tool * surface; `cli` = the real Supabase CLI inside a local-stack Docker sandbox. + * `rest` = direct HTTP access to the hosted API. * Required on every scenario, so each one is categorized on this dimension * rather than silently exported without a value. * @@ -98,7 +100,7 @@ export type ExperimentDisplayMetadata = z.infer< * tools runtime. `interface: cli` additionally forces a sandbox for scenarios * that start from an empty workspace (no `local/`). */ -export const evalInterfaceSchema = z.enum(['mcp', 'cli']); +export const evalInterfaceSchema = z.enum(['mcp', 'cli', 'rest']); export const EVAL_INTERFACES = evalInterfaceSchema.options; export type EvalInterface = z.infer; // Matches VERSION_RE in packages/sandbox/src/cli-channel.ts, so a resolved diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 895383bb6..e0c66961c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -961,6 +961,8 @@ export type EvalSessionArgs = { export type EvalSession = { mcpServers: Record; + /** Connection settings available to every process in the agent sandbox. */ + env?: Record; /** * Text for the agent's system prompt, joined from the MCP servers' * `promptAddendum`s. Must be empty for a CLI agent, which runs with its own diff --git a/packages/sandbox/src/bare-sandbox.ts b/packages/sandbox/src/bare-sandbox.ts index fa8895782..6b6772d08 100644 --- a/packages/sandbox/src/bare-sandbox.ts +++ b/packages/sandbox/src/bare-sandbox.ts @@ -12,6 +12,7 @@ export interface BareSandboxHandle { } export interface BareSandboxOptions { + env?: Record; /** Supabase CLI version baked into the sandbox image. */ cliVersion?: string; /** Skills to install into the sandbox. */ @@ -38,6 +39,7 @@ export async function createBareSandbox( options: BareSandboxOptions ): Promise { const env = await createAgentEnvironment({ + env: options.env, cliVersion: options.cliVersion, skills: options.skills, mounts: options.mounts, diff --git a/packages/sandbox/test/unit.test.ts b/packages/sandbox/test/unit.test.ts index 16e723c12..50db7dec3 100644 --- a/packages/sandbox/test/unit.test.ts +++ b/packages/sandbox/test/unit.test.ts @@ -604,6 +604,9 @@ describe('interface frontmatter', () => { expect( parseEvalMarkdown(buildMarkdown('interface: mcp')).metadata.interface ).toBe('mcp'); + expect( + parseEvalMarkdown(buildMarkdown('interface: rest')).metadata.interface + ).toBe('rest'); }); it('rejects a scenario with no interface, naming the source file', () => { @@ -613,9 +616,9 @@ describe('interface frontmatter', () => { }); it('rejects an unknown interface', () => { - expect(() => parseEvalMarkdown(buildMarkdown('interface: rest'))).toThrow( - /interface/ - ); + expect(() => + parseEvalMarkdown(buildMarkdown('interface: unknown')) + ).toThrow(/interface/); }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a4497a57f..34b1e010f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,6 +71,12 @@ importers: '@ai-sdk/openai': specifier: 'catalog:' version: 3.0.69(zod@4.6.7) + '@supabase-evals/platform-lite': + specifier: workspace:* + version: link:packages/platform-lite + '@supabase/supabase-js': + specifier: 'catalog:' + version: 2.108.1 common-tags: specifier: ^1.8.2 version: 1.8.2 @@ -80,6 +86,9 @@ importers: smol-toml: specifier: ^1.9.1 version: 1.9.1 + zod: + specifier: 'catalog:' + version: 4.6.7 devDependencies: '@biomejs/biome': specifier: 1.9.4 @@ -102,6 +111,9 @@ importers: typescript: specifier: 'catalog:' version: 5.9.3 + vitest: + specifier: 'catalog:' + version: 5.0.3(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@21.0.4)(vite@7.3.7(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) apps/framework: dependencies: diff --git a/skills/spark/SKILL.md b/skills/spark/SKILL.md new file mode 100644 index 000000000..c980c3af5 --- /dev/null +++ b/skills/spark/SKILL.md @@ -0,0 +1,50 @@ +--- +name: spark +description: Spark REST API for database schema, data, and email/password authentication. +--- + +Base URL: `$SPARK_PROJECT_URL`. Send the API key in the `apikey` header, not a query parameter. `$SPARK_SECRET_KEY` acts as `service_role`; `$SPARK_PUBLISHABLE_KEY` acts as `anon`. For a user session, also send `Authorization: Bearer `. Keep keys and tokens out of output. + +Use `Content-Type: application/json` for JSON bodies. Error responses contain diagnostic details in the body. + +## Schema and SQL + +| Method | Path | Request / behavior | +| --- | --- | --- | +| GET | `/system/v1/migrations` | Applied migrations, oldest first, including their SQL statements. Secret key required. | +| POST | `/system/v1/migrations` | `{"version":"...","name":"...","sql":"..."}`. Applies SQL and reloads the Data API schema cache. Use an unused version. Secret key required. | +| POST | `/database/query` | `{"query":"SELECT ...","parameters":[]}`. Read-only SQL inspection. Secret key required. | + +Apply DDL through migrations and DML through the Data API. SQL compatibility is partial: `information_schema.tables` and `information_schema.columns` are unavailable. Inspect schema through migration statements or `SELECT * FROM pragma_table_info('')`. + +## Data API + +Tables in `public` are available at `/rest/v1/
`. + +| Method | Behavior | +| --- | --- | +| GET, HEAD | Read rows or headers. Supports `select`, filters such as `id=eq.1`, `order`, `limit`, and `offset`. | +| POST | Insert one JSON object or an array of rows. | +| PATCH | Update rows selected by query-string filters. | +| DELETE | Delete rows selected by query-string filters. | + +`Prefer: return=representation` returns inserted or updated rows. `Prefer: count=exact` requests a count. `Accept: application/vnd.pgrst.object+json` requests one row as an object. + +## Auth + +Email/password only. Signup confirms the email immediately without sending mail. Signup and token requests return `access_token`, `refresh_token`, and `user`. + +| Method | Path | Request / behavior | +| --- | --- | --- | +| POST | `/auth/v1/signup` | `{"email":"...","password":"..."}`. Creates a user and session. | +| POST | `/auth/v1/token?grant_type=password` | `{"email":"...","password":"..."}`. Signs in. | +| POST | `/auth/v1/token?grant_type=refresh_token` | `{"refresh_token":"..."}`. Refreshes a session. | +| GET | `/auth/v1/user` | Current user; requires the session bearer token. | +| POST | `/auth/v1/logout` | Ends the session; requires the session bearer token. | + +## Limits + +- No row-level security: publishable keys can read and write every row. RLS statements and policies are unsupported. +- No OpenAPI root (`GET /rest/v1/`), RPC, or views. +- Upsert and PUT on an existing row with an integer primary key are known to fail. +- No OTP, magic links, OAuth, password recovery, or MFA. A project secret key does not act as an Auth admin session.