Add necessary flags to dump optimized inlining info and fix walltime marker placement

.github/workflows/ci.yml

@@ -51,6 +51,12 @@ jobs:
       matrix:
         node-version: ["18", "20.5.1"]
         example: ${{ fromJson(needs.list-examples.outputs.examples) }}
+        # tinybench v5+ requires Node >=20, so its examples cannot run on Node 18
+        exclude:
+          - node-version: "18"
+            example: "with-tinybench-v5"
+          - node-version: "18"
+            example: "with-tinybench-v6"
       fail-fast: false
     steps:
       - uses: "actions/checkout@v4"

.github/workflows/codspeed.yml

@@ -82,8 +82,12 @@ jobs:
       - name: Run benchmarks
         # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2`
         uses: CodSpeedHQ/action@main
+        env:
+          CODSPEED_WALLTIME_PROFILER: samply
         with:
           mode: walltime
+          # TODO: Remove this once runner is released
+          runner-version: v4.17.7-alpha.2
           run: |
             pnpm turbo run bench --filter=@codspeed/tinybench-plugin
             pnpm turbo run bench --filter=@codspeed/vitest-plugin
@@ -173,3 +177,4 @@ jobs:
           working-directory: examples/with-electron-and-walltime
           run: xvfb-run -a pnpm bench:electron
           mode: walltime
+          runner-version: v4.17.7-alpha.2

CLAUDE.md

@@ -88,4 +88,10 @@ Based on the codebase analysis, to add stats access features:
 ## Repository Management Memories
 
 - Use pnpm instead of npm
-- To run tests in a package use moon <package-name>:test
+- The monorepo uses [Turborepo](https://turborepo.com). Run a task for a single
+  package with a filter, using the package's full name (e.g. `@codspeed/tinybench-plugin`):
+  - Test: `pnpm turbo run test --filter=<package-name>`
+  - Build: `pnpm turbo run build --filter=<package-name>`
+  - Typecheck: `pnpm turbo run typecheck --filter=<package-name>`
+  - Lint: `pnpm turbo run lint --filter=<package-name>`
+  - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`).

examples/with-tinybench-v5/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "with-tinybench-v5",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "bench-tinybench": "node --loader esbuild-register/loader -r esbuild-register tinybench.ts",
+    "typecheck": "tsc --noEmit --pretty"
+  },
+  "devDependencies": {
+    "@codspeed/tinybench-plugin": "workspace:*",
+    "esbuild-register": "^3.4.2",
+    "tinybench": "^5.1.0",
+    "typescript": "^5.1.3"
+  }
+}

examples/with-tinybench-v5/tinybench.ts

@@ -0,0 +1,39 @@
+import { withCodSpeed } from "@codspeed/tinybench-plugin";
+import { Bench } from "tinybench";
+
+function recursiveFibonacci(n: number): number {
+  if (n < 2) {
+    return n;
+  }
+  return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2);
+}
+
+// The plugin resolves tinybench's `Bench` type from the version hoisted at the
+// monorepo root, which differs from the major pinned in this example. A real
+// consumer has a single installed tinybench, so this mismatch is local to the
+// workspace; `@ts-expect-error` keeps the example typechecking and flags us if
+// the discrepancy ever resolves on its own.
+// @ts-expect-error cross-version Bench type mismatch within the monorepo
+const bench = withCodSpeed(new Bench({ time: 100, warmup: true }));
+
+bench
+  .add("recursive fibo 10", () => {
+    recursiveFibonacci(10);
+  })
+  // Register the per-task hooks so the plugin's handling of them is exercised.
+  .add(
+    "recursive fibo 15 with hooks",
+    () => {
+      recursiveFibonacci(15);
+    },
+    {
+      beforeAll() {},
+      beforeEach() {},
+      afterEach() {},
+      afterAll() {},
+    },
+  );
+
+bench.run().then(() => {
+  console.table(bench.table());
+});

examples/with-tinybench-v5/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "compilerOptions": {
+    "lib": ["es2023"],
+    "module": "ESNext",
+    "target": "es2022",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "moduleResolution": "Node"
+  }
+}

examples/with-tinybench-v6/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "with-tinybench-v6",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "bench-tinybench": "node --loader esbuild-register/loader -r esbuild-register tinybench.ts",
+    "typecheck": "tsc --noEmit --pretty"
+  },
+  "devDependencies": {
+    "@codspeed/tinybench-plugin": "workspace:*",
+    "esbuild-register": "^3.4.2",
+    "tinybench": "^6.0.2",
+    "typescript": "^5.1.3"
+  }
+}

examples/with-tinybench-v6/tinybench.ts

@@ -0,0 +1,39 @@
+import { withCodSpeed } from "@codspeed/tinybench-plugin";
+import { Bench } from "tinybench";
+
+function recursiveFibonacci(n: number): number {
+  if (n < 2) {
+    return n;
+  }
+  return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2);
+}
+
+// The plugin resolves tinybench's `Bench` type from the version hoisted at the
+// monorepo root, which differs from the major pinned in this example. A real
+// consumer has a single installed tinybench, so this mismatch is local to the
+// workspace; `@ts-expect-error` keeps the example typechecking and flags us if
+// the discrepancy ever resolves on its own.
+// @ts-expect-error cross-version Bench type mismatch within the monorepo
+const bench = withCodSpeed(new Bench({ time: 100, warmup: true }));
+
+bench
+  .add("recursive fibo 10", () => {
+    recursiveFibonacci(10);
+  })
+  // Register the per-task hooks so the plugin's handling of them is exercised.
+  .add(
+    "recursive fibo 15 with hooks",
+    () => {
+      recursiveFibonacci(15);
+    },
+    {
+      beforeAll() {},
+      beforeEach() {},
+      afterEach() {},
+      afterAll() {},
+    },
+  );
+
+bench.run().then(() => {
+  console.table(bench.table());
+});

examples/with-tinybench-v6/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "compilerOptions": {
+    "lib": ["es2023"],
+    "module": "ESNext",
+    "target": "es2022",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "moduleResolution": "Node"
+  }
+}

packages/core/src/index.ts

@@ -56,7 +56,10 @@ export const setupCore = () => {
   }
 
   native_core.InstrumentHooks.setIntegration("codspeed-node", __VERSION__);
-  linuxPerf.start();
+  // In walltime, we use node's native profiling options rather than our own, cf `getV8Flags`
+  if (getCodspeedRunnerMode() !== "walltime") {
+    linuxPerf.start();
+  }
   checkV8Flags();
 
   // Collect Node.js runtime environment to detect changes that could
@@ -68,7 +71,9 @@ export const setupCore = () => {
 };
 
 export const teardownCore = () => {
-  linuxPerf.stop();
+  if (getCodspeedRunnerMode() !== "walltime") {
+    linuxPerf.stop();
+  }
 };
 
 export type {

packages/core/src/introspection.ts

@@ -1,32 +1,56 @@
 import { writeFileSync } from "fs";
+import path from "path";
+
 import { getInstrumentMode } from ".";
 
 const CUSTOM_INTROSPECTION_EXIT_CODE = 0;
 
+const V8_LOG_FILENAME_PATTERN = "codspeed-v8-%p.log";
+
 export const getV8Flags = () => {
   const nodeVersionMajor = parseInt(process.version.slice(1).split(".")[0]);
   const instrumentMode = getInstrumentMode();
 
   const flags = ["--interpreted-frames-native-stack", "--allow-natives-syntax"];
 
-  if (instrumentMode === "analysis") {
-    flags.push(
-      ...[
-        "--hash-seed=1",
-        "--random-seed=1",
-        "--no-opt",
-        "--predictable",
-        "--predictable-gc-schedule",
-        "--expose-gc",
-        "--no-concurrent-sweeping",
-        "--max-old-space-size=4096",
-      ],
-    );
-    if (nodeVersionMajor < 18) {
-      flags.push("--no-randomize-hashes");
+  switch (instrumentMode) {
+    case "analysis": {
+      flags.push(
+        ...[
+          "--hash-seed=1",
+          "--random-seed=1",
+          "--no-opt",
+          "--predictable",
+          "--predictable-gc-schedule",
+          "--expose-gc",
+          "--no-concurrent-sweeping",
+          "--max-old-space-size=4096",
+        ],
+      );
+      if (nodeVersionMajor < 18) {
+        flags.push("--no-randomize-hashes");
+      }
+      if (nodeVersionMajor < 20) {
+        flags.push("--no-scavenge-task");
+      }
+
+      break;
     }
-    if (nodeVersionMajor < 20) {
-      flags.push("--no-scavenge-task");
+
+    case "walltime": {
+      // Emit the V8 jitdump
+      flags.push("--perf-prof");
+      const v8LogDir = process.env.CODSPEED_V8_LOG;
+      if (v8LogDir) {
+        flags.push(
+          ...[
+            "--log-code",
+            "--no-log-source-code",
+            "--no-logfile-per-isolate",
+            `--logfile=${path.join(v8LogDir, V8_LOG_FILENAME_PATTERN)}`,
+          ],
+        );
+      }
     }
   }
 

packages/tinybench-plugin/src/analysis.ts

@@ -3,10 +3,11 @@ import {
   InstrumentHooks,
   mongoMeasurement,
   optimizeFunction,
+  optimizeFunctionSync,
   wrapWithRootFrame,
   wrapWithRootFrameSync,
 } from "@codspeed/core";
-import { Bench, Fn, FnOptions, Task } from "tinybench";
+import { Bench, Task } from "tinybench";
 import { BaseBenchRunner } from "./shared";
 
 export function setupCodspeedAnalysisBench(
@@ -28,7 +29,7 @@ class AnalysisBenchRunner extends BaseBenchRunner {
   }
 
   protected async runTaskAsync(task: Task, uri: string): Promise<void> {
-    const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn };
+    const { fnOpts, fn } = this.getTaskData(task);
 
     await fnOpts?.beforeAll?.call(task, "run");
     await optimizeFunction(async () => {
@@ -40,7 +41,14 @@ class AnalysisBenchRunner extends BaseBenchRunner {
     await mongoMeasurement.start(uri);
 
     global.gc?.();
-    await this.wrapWithInstrumentHooksAsync(wrapWithRootFrame(fn), uri);
+    const rootFrame = wrapWithRootFrame(fn);
+    InstrumentHooks.startBenchmark();
+    try {
+      await rootFrame();
+    } finally {
+      InstrumentHooks.stopBenchmark();
+      InstrumentHooks.setExecutedBenchmark(process.pid, uri);
+    }
 
     await mongoMeasurement.stop(uri);
     await fnOpts?.afterEach?.call(task, "run");
@@ -50,12 +58,27 @@ class AnalysisBenchRunner extends BaseBenchRunner {
   }
 
   protected runTaskSync(task: Task, uri: string): void {
-    const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn };
+    const { fnOpts, fn } = this.getTaskData(task);
 
     fnOpts?.beforeAll?.call(task, "run");
+
+    optimizeFunctionSync(async () => {
+      fnOpts?.beforeEach?.call(task, "run");
+      await fn();
+      fnOpts?.afterEach?.call(task, "run");
+    });
+
     fnOpts?.beforeEach?.call(task, "run");
 
-    this.wrapWithInstrumentHooks(wrapWithRootFrameSync(fn), uri);
+    global.gc?.();
+    const rootFrame = wrapWithRootFrameSync(fn);
+    InstrumentHooks.startBenchmark();
+    try {
+      rootFrame();
+    } finally {
+      InstrumentHooks.stopBenchmark();
+      InstrumentHooks.setExecutedBenchmark(process.pid, uri);
+    }
 
     fnOpts?.afterEach?.call(task, "run");
     fnOpts?.afterAll?.call(task, "run");

packages/tinybench-plugin/src/benchOptions.ts

@@ -0,0 +1,24 @@
+import { Bench, Hook } from "tinybench";
+
+// The benchmark options tinybench resolves: timing knobs plus the suite-level
+// setup/teardown hooks. tinybench guarantees `setup`/`teardown` are populated
+// (at least with no-op defaults), so they are non-optional here.
+export interface ResolvedBenchOptions {
+  setup: Hook;
+  teardown: Hook;
+  warmup?: boolean;
+  warmupIterations?: number;
+  warmupTime?: number;
+  iterations?: number;
+  time?: number;
+  retainSamples?: boolean;
+}
+
+// Up to tinybench v5 the resolved options lived under `bench.opts`; from v6 they
+// are flattened onto the bench instance itself. Returning the holder object
+// (rather than a copy) keeps in-place mutation of `setup`/`teardown` working,
+// which the walltime runner relies on to bracket its instrumentation window.
+export function getBenchOptions(bench: Bench): ResolvedBenchOptions {
+  const opts = (bench as unknown as { opts?: ResolvedBenchOptions }).opts;
+  return opts ?? (bench as unknown as ResolvedBenchOptions);
+}