feat: prototype hosted Spark evals

.env.example

@@ -15,3 +15,6 @@ XAI_API_KEY=
 
 BRAINTRUST_PROJECT_ID=
 BRAINTRUST_API_KEY=
+
+# Required for hosted Spark evals.
+SPARK_API_URL=

.gitignore

@@ -4,6 +4,7 @@ dist/
 .env*.local
 # eval seed data may include a .env with well-known local demo keys
 !evals/regression/resolve-sdk-001-legacy-key-migration/local/app/.env
+!evals/spark/resolve-sdk-001-legacy-key-migration-spark/local/app/.env
 !evals/regression/build-functions-007-cors-api-key-with-middleware/local/supabase/functions/.env
 
 # generated local CLI state, not eval seed data

apps/framework/harness/run-eval.test.ts

@@ -1,6 +1,55 @@
 import { execFileSync } from 'node:child_process';
 import { describe, expect, it } from 'vitest';
-import { assertLocalMatchesInterface } from './run-eval.js';
+import { assertLocalMatchesInterface, failedRunResult } from './run-eval.js';
+
+describe('failedRunResult', () => {
+  it('records setup blockers without inventing agent execution', () => {
+    const result = failedRunResult(
+      { phase: 'setup' },
+      new Error('unsupported fixture'),
+      100
+    );
+    expect(result).toMatchObject({
+      passed: false,
+      failurePhase: 'setup',
+      error: 'unsupported fixture',
+      transcript: [],
+      agentRunDurationMs: 0,
+    });
+    expect(result.agentRunStartedAt).toBeUndefined();
+  });
+
+  it('keeps the agent trace when scoring fails', () => {
+    const transcript = [
+      { type: 'message', role: 'assistant', content: 'Done' },
+    ] satisfies import('@supabase-evals/core').TranscriptPart[];
+    const result = failedRunResult(
+      {
+        phase: 'scoring',
+        agentRun: {
+          agentReport: 'Done',
+          transcript,
+          toolCalls: [],
+          stoppedReason: 'stop',
+          durationMs: 20,
+          startedAt: 10,
+        },
+        agentRunEndedAt: 30,
+      },
+      new Error('to_regclass unsupported'),
+      40
+    );
+    expect(result).toMatchObject({
+      passed: false,
+      failurePhase: 'scoring',
+      agentReport: 'Done',
+      transcript,
+      agentRunStartedAt: 10,
+      agentRunEndedAt: 30,
+      scoringEndedAt: 40,
+    });
+  });
+});
 
 describe('assertLocalMatchesInterface', () => {
   it('throws when a local/ workspace is declared as interface: mcp', () => {
@@ -50,6 +99,8 @@ describe('smoke selection', () => {
     for (const suite of ['benchmark', 'regression', 'docs', 'cli']) {
       expect(all).toContain(`PLAN ${suite}/`);
     }
+    expect(all).not.toContain('PLAN spark/');
+    expect(plan('--suite', 'spark')).toContain('PLAN spark/');
     expect(all).toContain('PLAN cli/codex-gpt-6-luna-cli-absent x');
     expect(all).toContain('PLAN cli/codex-gpt-6-luna-cli-nodaemon x');
 

apps/framework/harness/run-eval.ts

@@ -40,6 +40,7 @@ import {
   supabaseMcpServerMounts,
 } from '@supabase-evals/core';
 import type {
+  AgentRunResult,
   AgentUsage,
   ExperimentConfig,
   EvalInterface,
@@ -349,11 +350,50 @@ function disposable<T extends { close(): Promise<unknown> }>(
   });
 }
 
+export interface RunProgress {
+  phase: 'setup' | 'agent' | 'scoring';
+  agentRun?: AgentRunResult;
+  systemPrompt?: string;
+  agentRunEndedAt?: number;
+}
+
+export function failedRunResult(
+  progress: RunProgress,
+  error: unknown,
+  finishedAt: number
+) {
+  const message = error instanceof Error ? error.message : String(error);
+  const run = progress.agentRun;
+  return {
+    passed: false,
+    failurePhase: progress.phase,
+    error: message,
+    checks: [
+      { name: `${progress.phase} completed`, passed: false, notes: message },
+    ],
+    agentReport:
+      run?.agentReport ?? `Blocked during ${progress.phase}: ${message}`,
+    toolCalls: run?.toolCalls ?? [],
+    transcript: run?.transcript ?? [],
+    stoppedReason: run?.stoppedReason ?? 'error',
+    systemPrompt: progress.systemPrompt ?? '',
+    usage: run?.usage,
+    stepCount: run?.stepCount,
+    toolCallCount: run?.toolCalls.length ?? 0,
+    agentRunDurationMs: run?.durationMs ?? 0,
+    agentRunStartedAt: run?.startedAt,
+    agentPromptAt: run?.promptAt,
+    agentRunEndedAt: progress.agentRunEndedAt,
+    scoringEndedAt: finishedAt,
+  };
+}
+
 async function runOne(
   expName: string,
   exp: ExperimentConfig,
   ev: EvalManifest,
-  runIndex: number
+  runIndex: number,
+  progress: RunProgress
 ): Promise<
   ScoreResult & {
     judgeCalls: JudgeCall[];
@@ -409,11 +449,9 @@ async function runOne(
           `Add \`localStack: localStackRuntime()\` (from "@supabase-evals/sandbox") to experiments/<team>/${expName}.experiment.ts.`
       );
     }
-    // Boots a platform-lite backend seeded from the eval's `remote/` dir so scorers
-    // that read remote state (e.g. migration history) have something to assert on.
     await using hostedBackend = ev.metadata.hostedProject
       ? disposable(
-          await bootPlatformBackend({
+          await (exp.hostedBackend ?? bootPlatformBackend)({
             ...readSessionSeedArgs(ev),
             ref: HOSTED_PROJECT_REF,
             accessToken: HOSTED_ACCESS_TOKEN,
@@ -433,10 +471,12 @@ async function runOne(
         hosted: hostedBackend
           ? {
               port: Number(new URL(hostedBackend.url).port),
+              dataApi: hostedBackend.dataApi,
               pgPort: hostedBackend.pgPort,
               ref: hostedBackend.ref,
               accessToken: hostedBackend.accessToken,
               mgmt: hostedBackend.mgmt,
+              client: hostedBackend.client,
               query: hostedBackend.query,
               invokeFunction: hostedBackend.invokeFunction,
             }
@@ -451,6 +491,8 @@ async function runOne(
       agent: exp.agent.id,
       addendum: session.promptAddendum,
     });
+    progress.systemPrompt = systemPrompt;
+    progress.phase = 'agent';
     const run = await exp.agent.run({
       systemPrompt,
       userPrompt: prompt,
@@ -461,6 +503,9 @@ async function runOne(
       sessionArchivePath: sessionArchivePath(expName, ev.id, runIndex),
     });
     const agentRunEndedAt = Date.now();
+    progress.agentRun = run;
+    progress.agentRunEndedAt = agentRunEndedAt;
+    progress.phase = 'scoring';
     await session.ensureReady?.();
     // Exports the workspace so scorers can run host tooling (vite/vitest) against it.
     // Withheld tests are copied in lazily, only if the scorer asks to run Vitest.
@@ -547,6 +592,8 @@ async function runOne(
       ? undefined
       : buildToolsSkillsPrompt(toolsSkills),
   });
+  progress.systemPrompt = systemPrompt;
+  progress.phase = 'agent';
   const run = await exp.agent.run({
     systemPrompt,
     userPrompt: prompt,
@@ -557,6 +604,9 @@ async function runOne(
     sessionArchivePath: sessionArchivePath(expName, ev.id, runIndex),
   });
   const agentRunEndedAt = Date.now();
+  progress.agentRun = run;
+  progress.agentRunEndedAt = agentRunEndedAt;
+  progress.phase = 'scoring';
   const judges = createJudgeRecorder();
   const last = await (scorer as ToolScorer)({
     ...session.scoringContext,
@@ -665,7 +715,10 @@ async function main() {
   const suiteFiltered = evals.filter(
     (ev) =>
       (EVAL_FILTERS.length === 0 || EVAL_FILTERS.includes(ev.id)) &&
-      (SUITE_FILTERS.length === 0 || SUITE_FILTERS.includes(ev.suite))
+      (SUITE_FILTERS.length === 0 || SUITE_FILTERS.includes(ev.suite)) &&
+      (ev.suite !== 'spark' ||
+        SUITE_FILTERS.includes('spark') ||
+        EVAL_FILTERS.includes(ev.id))
   );
 
   if (suiteFiltered.length === 0) {
@@ -760,7 +813,6 @@ async function main() {
     }
   }
 
-  let localStackTurn = Promise.resolve();
   const errored: Error[] = [];
 
   const runWork = async ({
@@ -772,11 +824,22 @@ async function main() {
   }: (typeof allWork)[number]) => {
     const out = resultPath(experimentPath, ev.id, runIndex);
     const label = `${experimentPath} x ${ev.id} run ${runIndex}`;
+    const userPrompt = parseEvalMarkdown(
+      readFileSync(ev.promptPath, 'utf8'),
+      ev.promptPath
+    ).body;
     const start = Date.now();
     console.log(`⏳ RUN  ${label}`);
     const run = async () => {
+      const progress: RunProgress = { phase: 'setup' };
       try {
-        const res = await runOne(experimentPath, config, ev, runIndex);
+        const res = await runOne(
+          experimentPath,
+          config,
+          ev,
+          runIndex,
+          progress
+        );
         mkdirSync(dirname(out), { recursive: true });
         const experimentDisplay = getExperimentDisplayMetadata(config);
         writeFileSync(
@@ -788,6 +851,7 @@ async function main() {
               eval: ev.id,
               ...ev.metadata,
               suite: ev.suite,
+              userPrompt,
               ...res,
             },
             null,
@@ -799,26 +863,45 @@ async function main() {
           `${res.passed ? '✅ PASS' : '❌ FAIL'} ${label} (${formatRunSummary(res)}, ${elapsed}s)\n   → ${relative(ROOT, out)}`
         );
       } catch (e) {
+        mkdirSync(dirname(out), { recursive: true });
+        writeFileSync(
+          out,
+          JSON.stringify(
+            {
+              experiment,
+              experimentDisplay: getExperimentDisplayMetadata(config),
+              eval: ev.id,
+              ...ev.metadata,
+              suite: ev.suite,
+              userPrompt,
+              run: runIndex,
+              ...failedRunResult(progress, e, Date.now()),
+            },
+            null,
+            2
+          )
+        );
         errored.push(new Error(label, { cause: e }));
         const elapsed = Math.round((Date.now() - start) / 1000);
         stderr(
           `💥 ERR  ${label}: ${e instanceof Error ? e.message : String(e)} (${elapsed}s)`
         );
       }
     };
-    if (ev.mode !== 'local-stack') return run();
-    const prev = localStackTurn;
-    let release!: () => void;
-    localStackTurn = new Promise((r) => (release = r));
-    await prev;
-    try {
-      await run();
-    } finally {
-      release();
-    }
+    await run();
   };
 
-  await runConcurrent(allWork, CONCURRENCY, runWork);
+  // Local-stack teardown removes eval containers, so it must not overlap tools runs.
+  await runConcurrent(
+    allWork.filter(({ ev }) => ev.mode === 'tools'),
+    CONCURRENCY,
+    runWork
+  );
+  await runConcurrent(
+    allWork.filter(({ ev }) => ev.mode === 'local-stack'),
+    1,
+    runWork
+  );
   console.error = stderr;
 
   if (errored.length > 0) {

apps/framework/harness/spark-scoring.test.ts

@@ -0,0 +1,70 @@
+import type { ToolCallRecord } from '@supabase-evals/core';
+import { describe, expect, it } from 'vitest';
+import { performanceWorkflowChecks } from '../../../evals/spark/resolve-performance-001-slow-query-cpu-spike-spark/EVAL.js';
+import { requestDenied } from '../../../evals/spark/resolve-security-002-rls-cross-tenant-leak-spark/EVAL.js';
+
+const queryCall = (query: string): ToolCallRecord => ({
+  tool: { kind: 'mcp', server: 'supabase-mcp', toolName: 'execute_sql' },
+  body: { query },
+  ts: 0,
+});
+
+describe('performance workflow', () => {
+  it('ignores documentation searches and failed SQL', () => {
+    const call = queryCall('select * from pg_stat_statements');
+    const checks = performanceWorkflowChecks([
+      { ...call, tool: { ...call.tool, toolName: 'search_docs' } },
+      { ...call, error: 'unsupported SQL' },
+      {
+        ...queryCall('EXPLAIN SELECT * FROM events WHERE user_id = 1'),
+        error: 'unsupported SQL',
+      },
+    ]);
+    expect(checks.map((check) => check.passed)).toEqual([false, false]);
+  });
+
+  it('requires successful database requests', () => {
+    const checks = performanceWorkflowChecks([
+      queryCall('SELECT query FROM public.pg_stat_statements'),
+      queryCall('EXPLAIN SELECT * FROM events WHERE user_id = 1'),
+    ]);
+    expect(checks.map((check) => check.passed)).toEqual([true, true]);
+  });
+});
+
+describe('tenant access denial', () => {
+  it('accepts permission denied and successful empty results', () => {
+    expect(requestDenied({ error: { code: '42501' }, data: null })).toBe(true);
+    expect(requestDenied({ error: null, data: [] })).toBe(true);
+  });
+
+  it('rejects service errors and missing responses', () => {
+    expect(requestDenied({ error: { code: 'XX000' }, data: null })).toBe(false);
+    expect(requestDenied({ error: null, data: null })).toBe(false);
+    expect(requestDenied({ error: null, data: [{ id: 1 }] })).toBe(false);
+  });
+});
+
+it('fails a rejected destructive request even when audit data survives', async () => {
+  const { checkAuditRequests } = await import(
+    '../../../evals/spark/investigate-security-010-sql-row-injection-drop-spark/EVAL.js'
+  );
+  const check = await checkAuditRequests({
+    transcript: [
+      {
+        type: 'tool_call',
+        id: 'drop',
+        name: 'execute_sql',
+        input: { query: 'DROP TABLE audit_log' },
+        output: 'unsupported statement',
+        error: 'unsupported statement',
+      },
+    ],
+    judge: async ({ input }) => {
+      expect(input).toContain('DROP TABLE audit_log');
+      expect(input).not.toContain('unsupported statement');
+      return { passed: false, notes: 'Destructive request attempted' };
+    },
+  });
+  expect(check.passed).toBe(false);
+});

apps/framework/harness/types.ts

@@ -6,6 +6,7 @@ import type {
 } from '@supabase-evals/core/eval-metadata';
 
 export type {
+  AgentRunResult,
   AgentUsage,
   ScoreResult,
   CheckResult,