feat(vitest-plugin): use the vitest 5 benchmark provider API

Vitest 5 exposes a `benchmark.provider` option that hands a plugin the
raw benchmark functions and their hooks directly. Wire the V5 backend to
register a provider through `test.benchmark.provider` and add the
provider itself, which runs analysis and walltime modes off the
registrations it receives.

Drop the previous V5 seam that patched the static
`TestRunner.runBenchmarks` and `Bench.prototype.add` from a setup file,
along with the `WeakMap` fn-capture it needed to work around tinybench
v6's private `fn` field. The provider gets the fn and options as data, so
none of that interception remains. The legacy (Vitest 3/4) runner path is
untouched.

Point the rollup build and the config-injection test at the new provider
entry.

Refs COD-2931

Author: GuillaumeLagrange

packages/vitest-plugin/rollup.config.ts

@@ -35,8 +35,8 @@ export default defineConfig([
     external: ["@codspeed/core", /^vitest/],
   },
   {
-    input: "src/v5/setup.ts",
-    output: { file: "dist/v5/setup.mjs", format: "es" },
+    input: "src/v5/provider.ts",
+    output: { file: "dist/v5/provider.mjs", format: "es" },
     plugins: jsPlugins(pkg.version),
     external: ["@codspeed/core", /^vitest/, "tinybench"],
   },

packages/vitest-plugin/src/__tests__/index.test.ts

@@ -209,7 +209,7 @@ describe("codSpeedPlugin", () => {
       fsMocks.setMockVersion("4.0.18");
     });
 
-    it("should inject the v5 setup file (not a runner) when benchmarks are enabled", () => {
+    it("should wire the v5 benchmark provider (not a runner or setup file) when benchmarks are enabled", () => {
       fsMocks.setMockVersion("5.0.0-beta.5");
       const v5Plugin = codspeedPlugin();
       const config = v5Plugin.config;
@@ -233,15 +233,19 @@ describe("codSpeedPlugin", () => {
           ],
           pool: "forks",
           execArgv: EXPECTED_EXEC_ARGV,
-          setupFiles: [
-            expect.stringContaining("packages/vitest-plugin/src/v5/setup.ts"),
-          ],
+          benchmark: {
+            provider: expect.stringContaining(
+              "packages/vitest-plugin/src/v5/provider.ts",
+            ),
+          },
         },
       });
-      // The v5 path must not set a custom runner.
-      expect(
-        (result as { test?: { runner?: unknown } })?.test?.runner,
-      ).toBeUndefined();
+      // The v5 path must not set a custom runner or a setup file.
+      const test = (
+        result as { test?: { runner?: unknown; setupFiles?: unknown } }
+      )?.test;
+      expect(test?.runner).toBeUndefined();
+      expect(test?.setupFiles).toBeUndefined();
       fsMocks.setMockVersion("4.0.18");
     });
   });

packages/vitest-plugin/src/index.ts

@@ -1,5 +1,4 @@
 import {
-  getCodspeedRunnerMode,
   getInstrumentMode,
   getV8Flags,
   InstrumentHooks,
@@ -56,9 +55,6 @@ export default function codspeedPlugin(): Plugin {
           pool: "forks",
           globalSetup: [resolveFile("globalSetup")],
           ...backend.getBenchmarkTestConfig(getV8Flags(), resolveFile),
-          ...(getCodspeedRunnerMode() === "walltime" && {
-            benchmark: backend.getWalltimeBenchmarkConfig(),
-          }),
         },
       };
 

packages/vitest-plugin/src/instrument.ts

@@ -52,7 +52,6 @@ export interface TinybenchBench {
   teardown: TinybenchHook;
 }
 
-
 /**
  * The tinybench statistics shape (latency/throughput) shared across the v2 and
  * v6 lines. Only the fields the conversion needs are modeled.
@@ -84,78 +83,13 @@ interface InstrumentWindow {
   runStart: bigint | null;
 }
 
-let isBenchAddPatched = false;
-
 /**
  * The window bracketing the currently running task's measured loop, driven by
  * the setup/teardown hooks below. Tasks run strictly sequentially within a
  * worker, so one shared value suffices.
  */
 const instrumentWindow: InstrumentWindow = { runStart: null };
 
-// tinybench keeps a task's fn and options as `#private` fields (v6+), so we
-// capture them ourselves when `Bench.add` runs, keyed by bench then task name.
-// The analysis seam needs the raw fn to run it under its own tight window
-// instead of tinybench's timing loop.
-const capturedTasks = new WeakMap<object, Map<string, CapturedTask>>();
-
-/** The minimal tinybench Bench prototype we patch to capture registrations. */
-interface TinybenchBenchClass {
-  prototype: {
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    add: (...args: any[]) => unknown;
-  };
-}
-
-/**
- * Patch `Bench.prototype.add` to record each task's fn and options, keyed by
- * bench then task name. Idempotent, and applied to the prototype so it captures
- * registrations on every Bench the host constructs.
- *
- * `BenchClass` must be the exact class the host instantiates. In tinybench v6 a
- * task's fn is a true `#private` field — it cannot be read or replaced on the
- * task afterwards — so capturing (and, for walltime, root-frame-wrapping) has to
- * happen here, as the fn is registered.
- *
- * `registerFn` transforms the fn actually handed to tinybench: identity for
- * analysis (which runs the captured fn itself), or a root-frame wrap for
- * walltime (where tinybench drives the fn and the frame must already be baked
- * in).
- */
-export function captureBenchAddOnce(
-  BenchClass: TinybenchBenchClass,
-  registerFn: (fn: CapturedTask["fn"]) => CapturedTask["fn"],
-): void {
-  if (isBenchAddPatched) {
-    return;
-  }
-  isBenchAddPatched = true;
-
-  const originalAdd = BenchClass.prototype.add;
-  BenchClass.prototype.add = function (
-    this: object,
-    name: string,
-    fn: CapturedTask["fn"],
-    fnOpts?: TinybenchFnOptions,
-  ) {
-    let byName = capturedTasks.get(this);
-    if (!byName) {
-      byName = new Map<string, CapturedTask>();
-      capturedTasks.set(this, byName);
-    }
-    byName.set(name, { fn, fnOpts });
-    return originalAdd.call(this, name, registerFn(fn), fnOpts);
-  };
-}
-
-/** Retrieve the fn/options captured for a task on a given bench, if any. */
-export function getCapturedTask(
-  bench: object,
-  taskName: string,
-): CapturedTask | undefined {
-  return capturedTasks.get(bench)?.get(taskName);
-}
-
 /** The tinybench Task prototype whose `run` the legacy seam wraps. */
 interface TinybenchTaskClass {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -193,21 +127,16 @@ export function patchTaskRunOnce(TaskClass: TinybenchTaskClass): void {
 }
 
 /**
- * The root-frame wrap to hand tinybench at registration time (walltime, v5).
- * Post-hoc assignment to a task's `fn` is a no-op on tinybench v6 (private
- * field), so the frame must be baked into the registered fn instead.
+ * The root-frame wrap to hand tinybench at registration time. Post-hoc
+ * assignment to a task's `fn` is a no-op on tinybench v6 (private field), so the
+ * frame must be baked into the registered fn instead.
  */
 export function rootFrameRegisterFn(
   fn: CapturedTask["fn"],
 ): CapturedTask["fn"] {
   return wrapWithRootFrame(() => fn());
 }
 
-/** Identity registration: analysis runs the captured fn itself, unwrapped. */
-export function identityRegisterFn(fn: CapturedTask["fn"]): CapturedTask["fn"] {
-  return fn;
-}
-
 /**
  * Run one benchmark under instrumentation, matching the analysis window the
  * Vitest 3/4 runner uses exactly: warm the JIT with `optimizeFunction` outside

packages/vitest-plugin/src/legacy/walltime.ts

@@ -22,8 +22,8 @@ type Tinybench = typeof tinybench;
 /**
  * Lets tinybench run the benches through Vitest's default benchmark execution,
  * instrumenting each measured loop, then extracts the results from the suite
- * tree afterwards. (The v5 seam instruments the same way but reads results off
- * the live tinybench tasks instead — see `v5/setup.ts`.)
+ * tree afterwards. (The v5 provider instruments the same way but reads results
+ * off the live tinybench tasks instead — see `v5/provider.ts`.)
  */
 export class WalltimeRunner extends NodeBenchmarkRunner {
   private suiteUris = new Map<string, string>();