From 1fb37efeb47857f645555cb6b6e392a5fe733665 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 18 Jun 2026 12:30:57 +0200 Subject: [PATCH 1/6] feat(core): emit V8 jitdump and code log for walltime profiling Add --perf-prof, --log-code, --no-log-source-code, --no-logfile-per-isolate and --logfile to getV8Flags in walltime mode. --perf-prof writes the jitdump samply uses to symbolicate JIT'd JS; --log-code writes the code log whose inlining map lets the symbolicating recover TurboFan/Maglev inlined frames the jitdump alone collapses. Source-code logging is left off to keep the log small. Refs COD-2821, COD-2822 --- packages/core/src/index.ts | 9 +++-- packages/core/src/introspection.ts | 58 +++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2fd780b5..1f23273f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -56,7 +56,10 @@ export const setupCore = () => { } native_core.InstrumentHooks.setIntegration("codspeed-node", __VERSION__); - linuxPerf.start(); + // In walltime, we use node's native profiling options rather than our own, cf `getV8Flags` + if (getCodspeedRunnerMode() !== "walltime") { + linuxPerf.start(); + } checkV8Flags(); // Collect Node.js runtime environment to detect changes that could @@ -68,7 +71,9 @@ export const setupCore = () => { }; export const teardownCore = () => { - linuxPerf.stop(); + if (getCodspeedRunnerMode() !== "walltime") { + linuxPerf.stop(); + } }; export type { diff --git a/packages/core/src/introspection.ts b/packages/core/src/introspection.ts index 77dcfed0..5bb232ec 100644 --- a/packages/core/src/introspection.ts +++ b/packages/core/src/introspection.ts @@ -1,32 +1,56 @@ import { writeFileSync } from "fs"; +import path from "path"; + import { getInstrumentMode } from "."; const CUSTOM_INTROSPECTION_EXIT_CODE = 0; +const V8_LOG_FILENAME_PATTERN = "codspeed-v8-%p.log"; + export const getV8Flags = () => { const nodeVersionMajor = parseInt(process.version.slice(1).split(".")[0]); const instrumentMode = getInstrumentMode(); const flags = ["--interpreted-frames-native-stack", "--allow-natives-syntax"]; - if (instrumentMode === "analysis") { - flags.push( - ...[ - "--hash-seed=1", - "--random-seed=1", - "--no-opt", - "--predictable", - "--predictable-gc-schedule", - "--expose-gc", - "--no-concurrent-sweeping", - "--max-old-space-size=4096", - ], - ); - if (nodeVersionMajor < 18) { - flags.push("--no-randomize-hashes"); + switch (instrumentMode) { + case "analysis": { + flags.push( + ...[ + "--hash-seed=1", + "--random-seed=1", + "--no-opt", + "--predictable", + "--predictable-gc-schedule", + "--expose-gc", + "--no-concurrent-sweeping", + "--max-old-space-size=4096", + ], + ); + if (nodeVersionMajor < 18) { + flags.push("--no-randomize-hashes"); + } + if (nodeVersionMajor < 20) { + flags.push("--no-scavenge-task"); + } + + break; } - if (nodeVersionMajor < 20) { - flags.push("--no-scavenge-task"); + + case "walltime": { + // Emit the V8 jitdump + flags.push("--perf-prof"); + const v8LogDir = process.env.CODSPEED_V8_LOG; + if (v8LogDir) { + flags.push( + ...[ + "--log-code", + "--no-log-source-code", + "--no-logfile-per-isolate", + `--logfile=${path.join(v8LogDir, V8_LOG_FILENAME_PATTERN)}`, + ], + ); + } } } From e07237e591b7ea5b2a94f88821d64dfbe511d9d5 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 18 Jun 2026 14:41:01 +0200 Subject: [PATCH 2/6] fix(tinybench-plugin): fix profiler capturing tinybench's internal functions --- CLAUDE.md | 8 +++- packages/tinybench-plugin/src/shared.ts | 28 ++++++++------ packages/tinybench-plugin/src/walltime.ts | 45 +++++++++++++++++++++-- 3 files changed, 64 insertions(+), 17 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 496e1bd9..b82671a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -88,4 +88,10 @@ Based on the codebase analysis, to add stats access features: ## Repository Management Memories - Use pnpm instead of npm -- To run tests in a package use moon :test \ No newline at end of file +- The monorepo uses [Turborepo](https://turborepo.com). Run a task for a single + package with a filter, using the package's full name (e.g. `@codspeed/tinybench-plugin`): + - Test: `pnpm turbo run test --filter=` + - Build: `pnpm turbo run build --filter=` + - Typecheck: `pnpm turbo run typecheck --filter=` + - Lint: `pnpm turbo run lint --filter=` + - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`). \ No newline at end of file diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 093ed0ec..b9cd9e8a 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -46,15 +46,11 @@ export abstract class BaseBenchRunner { } protected wrapWithInstrumentHooks(fn: () => T, uri: string): T { - InstrumentHooks.startBenchmark(); - const runStart = InstrumentHooks.currentTimestamp(); + const runStart = this.openInstrumentWindow(); try { return fn(); } finally { - const runEnd = InstrumentHooks.currentTimestamp(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - this.sendBenchmarkMarkers(runStart, runEnd); + this.closeInstrumentWindow(uri, runStart); } } @@ -62,18 +58,26 @@ export abstract class BaseBenchRunner { fn: Fn, uri: string, ): Promise { - InstrumentHooks.startBenchmark(); - const runStart = InstrumentHooks.currentTimestamp(); + const runStart = this.openInstrumentWindow(); try { return await fn(); } finally { - const runEnd = InstrumentHooks.currentTimestamp(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - this.sendBenchmarkMarkers(runStart, runEnd); + this.closeInstrumentWindow(uri, runStart); } } + protected openInstrumentWindow(): bigint { + InstrumentHooks.startBenchmark(); + return InstrumentHooks.currentTimestamp(); + } + + protected closeInstrumentWindow(uri: string, runStart: bigint): void { + const runEnd = InstrumentHooks.currentTimestamp(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + this.sendBenchmarkMarkers(runStart, runEnd); + } + protected abstract getModeName(): string; protected abstract runTaskAsync(task: Task, uri: string): Promise; protected abstract runTaskSync(task: Task, uri: string): void; diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 9944a61a..81ab5d5f 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -9,7 +9,7 @@ import { type BenchmarkStats, type Benchmark as CodspeedBenchmark, } from "@codspeed/core"; -import { Bench, Fn, Task, TaskResult } from "tinybench"; +import { Bench, Fn, Hook, Task, TaskResult } from "tinybench"; import { BaseBenchRunner } from "./shared"; export function setupCodspeedWalltimeBench( @@ -17,16 +17,52 @@ export function setupCodspeedWalltimeBench( rootCallingFile: string, ): void { const runner = new WalltimeBenchRunner(bench, rootCallingFile); + runner.installInstrumentHooks(); runner.setupBenchMethods(); } class WalltimeBenchRunner extends BaseBenchRunner { private codspeedBenchmarks: CodspeedBenchmark[] = []; + // Carries the window start timestamp from the setup hook to the teardown + // hook. Tasks run strictly sequentially, so a single field is enough. + private runStart: bigint | null = null; + protected getModeName(): string { return "walltime mode"; } + /** + * Drive the instrumentation window from the task's setup/teardown hooks so it + * brackets only tinybench's measured loop, excluding warmup and the + * statistics computation (`processRunResult`) that surround it. The user's + * own hooks are preserved and still run in their original order relative to + * the work under test. + */ + public installInstrumentHooks(): void { + // `bench.opts` is typed `Readonly`, but tinybench mutates it at runtime and + // always resolves `setup`/`teardown` to (at least) a no-op default. + const opts = this.bench.opts as { setup: Hook; teardown: Hook }; + const userSetup = opts.setup; + const userTeardown = opts.teardown; + + opts.setup = (task, mode) => { + const setupResult = userSetup(task, mode); + if (mode === "run") { + this.runStart = this.openInstrumentWindow(); + } + return setupResult; + }; + + opts.teardown = (task, mode) => { + if (mode === "run" && task) { + this.closeInstrumentWindow(this.getTaskUri(task), this.runStart!); + this.runStart = null; + } + return userTeardown(task, mode); + }; + } + protected async runTaskAsync(task: Task, uri: string): Promise { // Override the function under test to add a static frame this.wrapTaskFunction(task, true); @@ -37,13 +73,14 @@ class WalltimeBenchRunner extends BaseBenchRunner { } await mongoMeasurement.start(uri); - await this.wrapWithInstrumentHooksAsync(() => task.run(), uri); + await task.run(); await mongoMeasurement.stop(uri); this.registerCodspeedBenchmarkFromTask(task); } - protected runTaskSync(task: Task, uri: string): void { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + protected runTaskSync(task: Task, _uri: string): void { // Override the function under test to add a static frame this.wrapTaskFunction(task, false); @@ -51,7 +88,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { task.warmup(); } - this.wrapWithInstrumentHooks(() => task.runSync(), uri); + task.runSync(); this.registerCodspeedBenchmarkFromTask(task); } From 2475d5b30215216348a47efde47be53521b668e4 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Sat, 20 Jun 2026 08:40:11 +0200 Subject: [PATCH 3/6] fix(vitest-plugin): bracket walltime window around the measured loop only The runner wrapped the entire tinybench Task.run() in the instrumentation window, folding the run-mode setup/teardown hooks and processRunResult (which sorts the latency samples twice and computes the full statistics) into the recorded sample. That framework overhead showed up in the walltime flamegraph. Drive the window from each bench's run-mode setup/teardown hooks instead, so it brackets only the measured loop. Warmup already runs as a separate Vitest call and is now excluded, and the post-loop statistics computation falls outside the window. User-provided hooks are preserved. The tinybench module namespace is frozen, so the instrumented Bench is handed back through a fresh module-shaped object that Vitest destructures from rather than reassigning the export. Closes COD-2925 Co-Authored-By: Claude --- packages/vitest-plugin/src/walltime/index.ts | 148 +++++++++++++++---- 1 file changed, 116 insertions(+), 32 deletions(-) diff --git a/packages/vitest-plugin/src/walltime/index.ts b/packages/vitest-plugin/src/walltime/index.ts index 92a09916..18c71215 100644 --- a/packages/vitest-plugin/src/walltime/index.ts +++ b/packages/vitest-plugin/src/walltime/index.ts @@ -6,6 +6,7 @@ import { wrapWithRootFrame, writeWalltimeResults, } from "@codspeed/core"; +import type * as tinybench from "tinybench"; import { RunnerTaskEventPack, RunnerTaskResultPack, @@ -15,6 +16,27 @@ import { NodeBenchmarkRunner } from "vitest/runners"; import { patchRootSuiteWithFullFilePath } from "../common"; import { extractBenchmarkResults } from "./utils"; +type Tinybench = typeof tinybench; + +/** A tinybench task, exposing the `fn` the runner wraps with the root frame. */ +interface TinybenchTask { + name: string; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fn: (...args: any[]) => any; +} + +/** tinybench's per-task setup/teardown hook signature. */ +type TinybenchHook = ( + task: TinybenchTask, + mode: "run" | "warmup", +) => Promise | void; + +/** The mutable subset of a tinybench Bench the runner reaches into. */ +interface TinybenchBench { + setup: TinybenchHook; + teardown: TinybenchHook; +} + /** * WalltimeRunner uses Vitest's default benchmark execution * and extracts results from the suite after completion @@ -24,6 +46,9 @@ export class WalltimeRunner extends NodeBenchmarkRunner { private suiteUris = new Map(); /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks private currentSuiteId: string | null = null; + // Carries the window start timestamp from the setup hook to the teardown + // hook. Tasks run strictly sequentially, so a single field is enough. + private runStart: bigint | null = null; async runSuite(suite: RunnerTestSuite): Promise { patchRootSuiteWithFullFilePath(suite); @@ -59,59 +84,118 @@ export class WalltimeRunner extends NodeBenchmarkRunner { } } - async importTinybench() { + private getBenchmarkUri(taskName: string): string { + if (this.currentSuiteId === null) { + throw new Error("currentSuiteId is null - something went wrong"); + } + const suiteUri = this.suiteUris.get(this.currentSuiteId) || ""; + return `${suiteUri}::${taskName}`; + } + + async importTinybench(): Promise { const tinybench = await super.importTinybench(); - if (this.isTinybenchHookedWithCodspeed) { - return tinybench; + // `tinybench` is a frozen ES module namespace, so the `Bench` export cannot + // be reassigned. Mutating the shared `Task.prototype` in place is allowed + // and only needs to happen once; the instrumented `Bench` is handed back + // through a fresh module-shaped object that Vitest destructures from. + if (!this.isTinybenchHookedWithCodspeed) { + this.isTinybenchHookedWithCodspeed = true; + this.patchTaskWithRootFrame(tinybench); } - this.isTinybenchHookedWithCodspeed = true; - const originalRun = tinybench.Task.prototype.run; - const pid = process.pid; - - const getSuiteUri = (): string => { - if (this.currentSuiteId === null) { - throw new Error("currentSuiteId is null - something went wrong"); - } - return this.suiteUris.get(this.currentSuiteId) || ""; + return { + ...tinybench, + Bench: this.createInstrumentedBench(tinybench), }; + } - tinybench.Task.prototype.run = async function () { - const suiteUri = getSuiteUri(); + /** + * Wrap each task's function with the root frame so collected stacks can be + * attributed to a benchmark. The window itself is driven by the bench's + * setup/teardown hooks (see createInstrumentedBench). + */ + private patchTaskWithRootFrame(tinybench: Tinybench): void { + const originalRun = tinybench.Task.prototype.run; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const task = this as any; + tinybench.Task.prototype.run = async function () { + const task = this as unknown as TinybenchTask; const originalFn = task.fn; task.fn = wrapWithRootFrame(() => originalFn.call(task)); - InstrumentHooks.startBenchmark(); - const runStart = InstrumentHooks.currentTimestamp(); try { await originalRun.call(this); } finally { - const runEnd = InstrumentHooks.currentTimestamp(); task.fn = originalFn; + } - // Benchmark markers must land inside the sample window opened by - // startBenchmark(), so they have to be emitted before stopBenchmark() - // closes it. The runner consumes the FIFO stream in order, so a marker - // sent after StopBenchmark falls outside the sample and breaks the - // expected SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, runStart); - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); + return this; + }; + } - InstrumentHooks.stopBenchmark(); + /** + * Drive the instrumentation window from each bench's run-mode setup/teardown + * hooks so it brackets only tinybench's measured loop, excluding the warmup + * that Vitest runs beforehand and the statistics computation tinybench + * performs after the loop. Wrapping the whole `Task.run()` would otherwise + * fold all of that framework overhead into the recorded sample. + * + * User-provided hooks are preserved and keep their order relative to the work + * under test. + */ + private createInstrumentedBench( + tinybench: Tinybench, + ): typeof tinybench.Bench { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const runner = this; + const OriginalBench = tinybench.Bench; + + class InstrumentedBench extends OriginalBench { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(...benchArgs: any[]) { + super(...benchArgs); + runner.installInstrumentHooks(this as unknown as TinybenchBench); + } + } - // Look up the URI by task name - const uri = `${suiteUri}::${this.name}`; - InstrumentHooks.setExecutedBenchmark(pid, uri); + return InstrumentedBench; + } + + private installInstrumentHooks(bench: TinybenchBench): void { + const userSetup = bench.setup; + const userTeardown = bench.teardown; + + bench.setup = async (task, mode) => { + await userSetup(task, mode); + if (mode === "run") { + InstrumentHooks.startBenchmark(); + this.runStart = InstrumentHooks.currentTimestamp(); } + }; - return this; + bench.teardown = async (task, mode) => { + if (mode === "run") { + this.closeInstrumentWindow(this.getBenchmarkUri(task.name)); + } + await userTeardown(task, mode); }; + } + + private closeInstrumentWindow(uri: string): void { + const runEnd = InstrumentHooks.currentTimestamp(); + const pid = process.pid; - return tinybench; + // Benchmark markers must land inside the sample window opened by + // startBenchmark(), so they have to be emitted before stopBenchmark() + // closes it. The runner consumes the FIFO stream in order, so a marker + // sent after StopBenchmark falls outside the sample and breaks the + // expected SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, this.runStart!); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); + + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(pid, uri); + this.runStart = null; } // Allow tinybench to retrieve the path to the currently running suite From 4601945698ebd67b7fec78f856eb4b09881824a4 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Sun, 21 Jun 2026 19:20:34 +0200 Subject: [PATCH 4/6] fix(tinybench-plugin): support tinybench v5 and v6 tinybench v6 moved the benchmark function, its options, and the resolved bench options behind ES private fields (and flattened the options onto the bench instance), which broke the plugin's reliance on reaching those internals through casts: analysis mode crashed with "fn is not a function" and walltime mode crashed reading `bench.opts.setup`. Capture the function and options ourselves when `bench.add` runs, normalize option access across the v4/v5 `bench.opts` layout and the v6 flattened one, bake the walltime root frame into the registered function instead of mutating the (now private) task function, and opt back into sample retention that v6 disabled by default. Add dedicated example fixtures pinning tinybench v5 and v6 so CI exercises every supported major, excluding them from the Node 18 leg since tinybench v5+ requires Node >=20. Closes COD-2929 Co-Authored-By: Claude Generated with AI Agent (Claude Code) --- .github/workflows/ci.yml | 6 +++ examples/with-tinybench-v5/package.json | 15 ++++++ examples/with-tinybench-v5/tinybench.ts | 39 +++++++++++++++ examples/with-tinybench-v5/tsconfig.json | 12 +++++ examples/with-tinybench-v6/package.json | 15 ++++++ examples/with-tinybench-v6/tinybench.ts | 39 +++++++++++++++ examples/with-tinybench-v6/tsconfig.json | 12 +++++ packages/tinybench-plugin/src/analysis.ts | 6 +-- packages/tinybench-plugin/src/benchOptions.ts | 24 +++++++++ packages/tinybench-plugin/src/index.ts | 15 +++++- packages/tinybench-plugin/src/shared.ts | 11 ++++ packages/tinybench-plugin/src/taskData.ts | 30 +++++++++++ packages/tinybench-plugin/src/walltime.ts | 50 ++++++++----------- pnpm-lock.yaml | 42 ++++++++++++++++ 14 files changed, 281 insertions(+), 35 deletions(-) create mode 100644 examples/with-tinybench-v5/package.json create mode 100644 examples/with-tinybench-v5/tinybench.ts create mode 100644 examples/with-tinybench-v5/tsconfig.json create mode 100644 examples/with-tinybench-v6/package.json create mode 100644 examples/with-tinybench-v6/tinybench.ts create mode 100644 examples/with-tinybench-v6/tsconfig.json create mode 100644 packages/tinybench-plugin/src/benchOptions.ts create mode 100644 packages/tinybench-plugin/src/taskData.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f87920b1..b6133882 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,12 @@ jobs: matrix: node-version: ["18", "20.5.1"] example: ${{ fromJson(needs.list-examples.outputs.examples) }} + # tinybench v5+ requires Node >=20, so its examples cannot run on Node 18 + exclude: + - node-version: "18" + example: "with-tinybench-v5" + - node-version: "18" + example: "with-tinybench-v6" fail-fast: false steps: - uses: "actions/checkout@v4" diff --git a/examples/with-tinybench-v5/package.json b/examples/with-tinybench-v5/package.json new file mode 100644 index 00000000..98a132cb --- /dev/null +++ b/examples/with-tinybench-v5/package.json @@ -0,0 +1,15 @@ +{ + "name": "with-tinybench-v5", + "private": true, + "type": "module", + "scripts": { + "bench-tinybench": "node --loader esbuild-register/loader -r esbuild-register tinybench.ts", + "typecheck": "tsc --noEmit --pretty" + }, + "devDependencies": { + "@codspeed/tinybench-plugin": "workspace:*", + "esbuild-register": "^3.4.2", + "tinybench": "^5.1.0", + "typescript": "^5.1.3" + } +} diff --git a/examples/with-tinybench-v5/tinybench.ts b/examples/with-tinybench-v5/tinybench.ts new file mode 100644 index 00000000..f2788bd0 --- /dev/null +++ b/examples/with-tinybench-v5/tinybench.ts @@ -0,0 +1,39 @@ +import { withCodSpeed } from "@codspeed/tinybench-plugin"; +import { Bench } from "tinybench"; + +function recursiveFibonacci(n: number): number { + if (n < 2) { + return n; + } + return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); +} + +// The plugin resolves tinybench's `Bench` type from the version hoisted at the +// monorepo root, which differs from the major pinned in this example. A real +// consumer has a single installed tinybench, so this mismatch is local to the +// workspace; `@ts-expect-error` keeps the example typechecking and flags us if +// the discrepancy ever resolves on its own. +// @ts-expect-error cross-version Bench type mismatch within the monorepo +const bench = withCodSpeed(new Bench({ time: 100, warmup: true })); + +bench + .add("recursive fibo 10", () => { + recursiveFibonacci(10); + }) + // Register the per-task hooks so the plugin's handling of them is exercised. + .add( + "recursive fibo 15 with hooks", + () => { + recursiveFibonacci(15); + }, + { + beforeAll() {}, + beforeEach() {}, + afterEach() {}, + afterAll() {}, + }, + ); + +bench.run().then(() => { + console.table(bench.table()); +}); diff --git a/examples/with-tinybench-v5/tsconfig.json b/examples/with-tinybench-v5/tsconfig.json new file mode 100644 index 00000000..7b8550e2 --- /dev/null +++ b/examples/with-tinybench-v5/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/examples/with-tinybench-v6/package.json b/examples/with-tinybench-v6/package.json new file mode 100644 index 00000000..0fe8d728 --- /dev/null +++ b/examples/with-tinybench-v6/package.json @@ -0,0 +1,15 @@ +{ + "name": "with-tinybench-v6", + "private": true, + "type": "module", + "scripts": { + "bench-tinybench": "node --loader esbuild-register/loader -r esbuild-register tinybench.ts", + "typecheck": "tsc --noEmit --pretty" + }, + "devDependencies": { + "@codspeed/tinybench-plugin": "workspace:*", + "esbuild-register": "^3.4.2", + "tinybench": "^6.0.2", + "typescript": "^5.1.3" + } +} diff --git a/examples/with-tinybench-v6/tinybench.ts b/examples/with-tinybench-v6/tinybench.ts new file mode 100644 index 00000000..f2788bd0 --- /dev/null +++ b/examples/with-tinybench-v6/tinybench.ts @@ -0,0 +1,39 @@ +import { withCodSpeed } from "@codspeed/tinybench-plugin"; +import { Bench } from "tinybench"; + +function recursiveFibonacci(n: number): number { + if (n < 2) { + return n; + } + return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); +} + +// The plugin resolves tinybench's `Bench` type from the version hoisted at the +// monorepo root, which differs from the major pinned in this example. A real +// consumer has a single installed tinybench, so this mismatch is local to the +// workspace; `@ts-expect-error` keeps the example typechecking and flags us if +// the discrepancy ever resolves on its own. +// @ts-expect-error cross-version Bench type mismatch within the monorepo +const bench = withCodSpeed(new Bench({ time: 100, warmup: true })); + +bench + .add("recursive fibo 10", () => { + recursiveFibonacci(10); + }) + // Register the per-task hooks so the plugin's handling of them is exercised. + .add( + "recursive fibo 15 with hooks", + () => { + recursiveFibonacci(15); + }, + { + beforeAll() {}, + beforeEach() {}, + afterEach() {}, + afterAll() {}, + }, + ); + +bench.run().then(() => { + console.table(bench.table()); +}); diff --git a/examples/with-tinybench-v6/tsconfig.json b/examples/with-tinybench-v6/tsconfig.json new file mode 100644 index 00000000..7b8550e2 --- /dev/null +++ b/examples/with-tinybench-v6/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index bdaca56b..7e742f10 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -6,7 +6,7 @@ import { wrapWithRootFrame, wrapWithRootFrameSync, } from "@codspeed/core"; -import { Bench, Fn, FnOptions, Task } from "tinybench"; +import { Bench, Task } from "tinybench"; import { BaseBenchRunner } from "./shared"; export function setupCodspeedAnalysisBench( @@ -28,7 +28,7 @@ class AnalysisBenchRunner extends BaseBenchRunner { } protected async runTaskAsync(task: Task, uri: string): Promise { - const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn }; + const { fnOpts, fn } = this.getTaskData(task); await fnOpts?.beforeAll?.call(task, "run"); await optimizeFunction(async () => { @@ -50,7 +50,7 @@ class AnalysisBenchRunner extends BaseBenchRunner { } protected runTaskSync(task: Task, uri: string): void { - const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn }; + const { fnOpts, fn } = this.getTaskData(task); fnOpts?.beforeAll?.call(task, "run"); fnOpts?.beforeEach?.call(task, "run"); diff --git a/packages/tinybench-plugin/src/benchOptions.ts b/packages/tinybench-plugin/src/benchOptions.ts new file mode 100644 index 00000000..a1ffef42 --- /dev/null +++ b/packages/tinybench-plugin/src/benchOptions.ts @@ -0,0 +1,24 @@ +import { Bench, Hook } from "tinybench"; + +// The benchmark options tinybench resolves: timing knobs plus the suite-level +// setup/teardown hooks. tinybench guarantees `setup`/`teardown` are populated +// (at least with no-op defaults), so they are non-optional here. +export interface ResolvedBenchOptions { + setup: Hook; + teardown: Hook; + warmup?: boolean; + warmupIterations?: number; + warmupTime?: number; + iterations?: number; + time?: number; + retainSamples?: boolean; +} + +// Up to tinybench v5 the resolved options lived under `bench.opts`; from v6 they +// are flattened onto the bench instance itself. Returning the holder object +// (rather than a copy) keeps in-place mutation of `setup`/`teardown` working, +// which the walltime runner relies on to bracket its instrumentation window. +export function getBenchOptions(bench: Bench): ResolvedBenchOptions { + const opts = (bench as unknown as { opts?: ResolvedBenchOptions }).opts; + return opts ?? (bench as unknown as ResolvedBenchOptions); +} diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index 64be2d8d..298f97b9 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -7,9 +7,11 @@ import { SetupInstrumentsRequestBody, SetupInstrumentsResponse, tryIntrospect, + wrapWithRootFrameSync, } from "@codspeed/core"; import { Bench } from "tinybench"; import { setupCodspeedAnalysisBench } from "./analysis"; +import { getOrCreateTaskDataMap } from "./taskData"; import { getOrCreateUriMap } from "./uri"; import { setupCodspeedWalltimeBench } from "./walltime"; @@ -22,9 +24,11 @@ export function withCodSpeed(bench: Bench): Bench { } const rootCallingFile = getCallingFile(1); + const instrumentMode = getInstrumentMode(); // Compute and register URI for bench const uriMap = getOrCreateUriMap(bench); + const taskDataMap = getOrCreateTaskDataMap(bench); const rawAdd = bench.add; bench.add = (name, fn, opts?) => { const callingFile = getCallingFile(1); @@ -34,10 +38,17 @@ export function withCodSpeed(bench: Bench): Bench { } uri += `::${name}`; uriMap.set(name, uri); - return rawAdd.bind(bench)(name, fn, opts); + taskDataMap.set(name, { fn, fnOpts: opts }); + // In walltime mode the task is driven by tinybench's own measured loop, so + // the root frame must be baked into the function tinybench stores rather + // than injected later (the function is an inaccessible private field on the + // task from tinybench v6 onwards). The sync wrapper is transparent to the + // function's return value, so it preserves tinybench's sync/async handling. + const registeredFn = + instrumentMode === "walltime" ? wrapWithRootFrameSync(fn) : fn; + return rawAdd.bind(bench)(name, registeredFn, opts); }; - const instrumentMode = getInstrumentMode(); if (instrumentMode === "analysis") { setupCodspeedAnalysisBench(bench, rootCallingFile); } else if (instrumentMode === "walltime") { diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index b9cd9e8a..d182f98c 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -7,6 +7,7 @@ import { teardownCore, } from "@codspeed/core"; import { Bench, Fn, Task } from "tinybench"; +import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; declare const __VERSION__: string; @@ -35,6 +36,16 @@ export abstract class BaseBenchRunner { return getTaskUri(this.bench, task.name, this.rootCallingFile); } + protected getTaskData(task: Task): CapturedTaskData { + const data = getTaskData(this.bench, task.name); + if (!data) { + throw new Error( + `[CodSpeed] No captured function found for task "${task.name}"`, + ); + } + return data; + } + protected logTaskCompletion(uri: string, status: string): void { console.log(`[CodSpeed] ${status} ${uri}`); } diff --git a/packages/tinybench-plugin/src/taskData.ts b/packages/tinybench-plugin/src/taskData.ts new file mode 100644 index 00000000..a5ecb2ab --- /dev/null +++ b/packages/tinybench-plugin/src/taskData.ts @@ -0,0 +1,30 @@ +import { Bench, Fn, FnOptions } from "tinybench"; + +// tinybench stores the benchmark function and its options as private fields on +// `Task`. They were reachable through casts on older majors but became true +// `#private` fields in v6, so we capture them ourselves when `bench.add` runs +// and key them by task name, mirroring the URI map. +export interface CapturedTaskData { + fn: Fn; + fnOpts?: FnOptions; +} + +const taskDataMap = new WeakMap>(); + +export function getOrCreateTaskDataMap( + bench: Bench, +): Map { + let map = taskDataMap.get(bench); + if (!map) { + map = new Map(); + taskDataMap.set(bench, map); + } + return map; +} + +export function getTaskData( + bench: Bench, + taskName: string, +): CapturedTaskData | undefined { + return taskDataMap.get(bench)?.get(taskName); +} diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 81ab5d5f..4963fc26 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -3,13 +3,12 @@ import { mongoMeasurement, msToNs, msToS, - wrapWithRootFrame, - wrapWithRootFrameSync, writeWalltimeResults, type BenchmarkStats, type Benchmark as CodspeedBenchmark, } from "@codspeed/core"; -import { Bench, Fn, Hook, Task, TaskResult } from "tinybench"; +import { Bench, Task, TaskResult } from "tinybench"; +import { getBenchOptions } from "./benchOptions"; import { BaseBenchRunner } from "./shared"; export function setupCodspeedWalltimeBench( @@ -40,9 +39,15 @@ class WalltimeBenchRunner extends BaseBenchRunner { * the work under test. */ public installInstrumentHooks(): void { - // `bench.opts` is typed `Readonly`, but tinybench mutates it at runtime and - // always resolves `setup`/`teardown` to (at least) a no-op default. - const opts = this.bench.opts as { setup: Hook; teardown: Hook }; + // The resolved options expose `setup`/`teardown` as typed `Readonly`, but + // tinybench populates them with (at least) no-op defaults and lets them be + // reassigned at runtime. + const opts = getBenchOptions(this.bench); + + // We build the walltime statistics from the per-round latency samples. + // tinybench stopped retaining them by default in v6, so opt back in. + opts.retainSamples = true; + const userSetup = opts.setup; const userTeardown = opts.teardown; @@ -64,11 +69,8 @@ class WalltimeBenchRunner extends BaseBenchRunner { } protected async runTaskAsync(task: Task, uri: string): Promise { - // Override the function under test to add a static frame - this.wrapTaskFunction(task, true); - // run the warmup of the task right before its actual run - if (this.bench.opts.warmup) { + if (getBenchOptions(this.bench).warmup) { await task.warmup(); } @@ -81,11 +83,8 @@ class WalltimeBenchRunner extends BaseBenchRunner { // eslint-disable-next-line @typescript-eslint/no-unused-vars protected runTaskSync(task: Task, _uri: string): void { - // Override the function under test to add a static frame - this.wrapTaskFunction(task, false); - - if (this.bench.opts.warmup) { - task.warmup(); + if (getBenchOptions(this.bench).warmup) { + task.warmupSync(); } task.runSync(); @@ -101,14 +100,6 @@ class WalltimeBenchRunner extends BaseBenchRunner { return this.finalizeWalltimeRun(false); } - private wrapTaskFunction(task: Task, isAsync: boolean): void { - const { fn } = task as unknown as { fn: Fn }; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (task as any).fn = isAsync - ? wrapWithRootFrame(fn) - : wrapWithRootFrameSync(fn); - } - private registerCodspeedBenchmarkFromTask(task: Task): void { const uri = this.getTaskUri(task); @@ -117,8 +108,9 @@ class WalltimeBenchRunner extends BaseBenchRunner { return; } - const warmupIterations = this.bench.opts.warmup - ? (this.bench.opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT) + const opts = getBenchOptions(this.bench); + const warmupIterations = opts.warmup + ? (opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT) : 0; const stats = convertTinybenchResultToBenchmarkStats( task.result, @@ -129,13 +121,11 @@ class WalltimeBenchRunner extends BaseBenchRunner { name: task.name, uri, config: { - max_rounds: this.bench.opts.iterations ?? null, - max_time_ns: this.bench.opts.time ? msToNs(this.bench.opts.time) : null, + max_rounds: opts.iterations ?? null, + max_time_ns: opts.time ? msToNs(opts.time) : null, min_round_time_ns: null, // tinybench does not have an option for this warmup_time_ns: - this.bench.opts.warmup && this.bench.opts.warmupTime - ? msToNs(this.bench.opts.warmupTime) - : null, + opts.warmup && opts.warmupTime ? msToNs(opts.warmupTime) : null, }, stats, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b2aeda91..d59ca9e2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -108,6 +108,36 @@ importers: specifier: ^4.0.1 version: 4.0.1 + examples/with-tinybench-v5: + devDependencies: + '@codspeed/tinybench-plugin': + specifier: workspace:* + version: link:../../packages/tinybench-plugin + esbuild-register: + specifier: ^3.4.2 + version: 3.4.2(esbuild@0.25.9) + tinybench: + specifier: ^5.1.0 + version: 5.1.0 + typescript: + specifier: ^5.1.3 + version: 5.8.3 + + examples/with-tinybench-v6: + devDependencies: + '@codspeed/tinybench-plugin': + specifier: workspace:* + version: link:../../packages/tinybench-plugin + esbuild-register: + specifier: ^3.4.2 + version: 3.4.2(esbuild@0.25.9) + tinybench: + specifier: ^6.0.2 + version: 6.0.2 + typescript: + specifier: ^5.1.3 + version: 5.8.3 + examples/with-typescript-cjs: devDependencies: '@codspeed/benchmark.js-plugin': @@ -5960,6 +5990,14 @@ packages: resolution: {integrity: sha512-Nb1srn7dvzkVx0J5h1vq8f48e3TIcbrS7e/UfAI/cDSef/n8yLh4zsAEsFkfpw6auTY+ZaspEvam/xs8nMnotQ==} engines: {node: '>=18.0.0'} + tinybench@5.1.0: + resolution: {integrity: sha512-LXKNtFualiKOm6gADe1UXPtf8+Nfn1CtPMEHAT33Fd2YjQatrujkDcK0+4wRC1X6t7fxUDXUs6BsvuIgfkDgDg==} + engines: {node: '>=20.0.0'} + + tinybench@6.0.2: + resolution: {integrity: sha512-FlHoQpcFvCzeXK5kVPvV7IVgW/hs/B36QWTz876iSdeJguBDfdTSRQmYmaHX+fQNt4hp+gEFB2XXw+8hT4/y8A==} + engines: {node: '>=20.0.0'} + tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} @@ -13495,6 +13533,10 @@ snapshots: tinybench@4.0.1: {} + tinybench@5.1.0: {} + + tinybench@6.0.2: {} + tinyexec@0.3.2: {} tinyexec@1.0.2: {} From 8f2de3c4f4324fbec86437df74e5b3ae90f4c485 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 19 Jun 2026 09:27:21 +0200 Subject: [PATCH 5/6] ci: use samply profiler for walltime integration tests --- .github/workflows/codspeed.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index d23925b5..e57462ec 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -82,8 +82,12 @@ jobs: - name: Run benchmarks # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` uses: CodSpeedHQ/action@main + env: + CODSPEED_WALLTIME_PROFILER: samply with: mode: walltime + # TODO: Remove this once runner is released + runner-version: v4.17.7-alpha.2 run: | pnpm turbo run bench --filter=@codspeed/tinybench-plugin pnpm turbo run bench --filter=@codspeed/vitest-plugin @@ -173,3 +177,4 @@ jobs: working-directory: examples/with-electron-and-walltime run: xvfb-run -a pnpm bench:electron mode: walltime + runner-version: v4.17.7-alpha.2 From 16bf96702208ca1ef8795ae554a1bd87f132e1ab Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Mon, 22 Jun 2026 14:19:45 +0200 Subject: [PATCH 6/6] feat: to fixup tinybench marker clean up --- packages/tinybench-plugin/src/analysis.ts | 27 ++++++++++- packages/tinybench-plugin/src/shared.ts | 56 +---------------------- packages/tinybench-plugin/src/walltime.ts | 26 ++++++++++- 3 files changed, 51 insertions(+), 58 deletions(-) diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 7e742f10..65bd6183 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -3,6 +3,7 @@ import { InstrumentHooks, mongoMeasurement, optimizeFunction, + optimizeFunctionSync, wrapWithRootFrame, wrapWithRootFrameSync, } from "@codspeed/core"; @@ -40,7 +41,14 @@ class AnalysisBenchRunner extends BaseBenchRunner { await mongoMeasurement.start(uri); global.gc?.(); - await this.wrapWithInstrumentHooksAsync(wrapWithRootFrame(fn), uri); + const rootFrame = wrapWithRootFrame(fn); + InstrumentHooks.startBenchmark(); + try { + await rootFrame(); + } finally { + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + } await mongoMeasurement.stop(uri); await fnOpts?.afterEach?.call(task, "run"); @@ -53,9 +61,24 @@ class AnalysisBenchRunner extends BaseBenchRunner { const { fnOpts, fn } = this.getTaskData(task); fnOpts?.beforeAll?.call(task, "run"); + + optimizeFunctionSync(async () => { + fnOpts?.beforeEach?.call(task, "run"); + await fn(); + fnOpts?.afterEach?.call(task, "run"); + }); + fnOpts?.beforeEach?.call(task, "run"); - this.wrapWithInstrumentHooks(wrapWithRootFrameSync(fn), uri); + global.gc?.(); + const rootFrame = wrapWithRootFrameSync(fn); + InstrumentHooks.startBenchmark(); + try { + rootFrame(); + } finally { + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + } fnOpts?.afterEach?.call(task, "run"); fnOpts?.afterAll?.call(task, "run"); diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index d182f98c..0fb811c5 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,12 +1,5 @@ -import { - getInstrumentMode, - InstrumentHooks, - MARKER_TYPE_BENCHMARK_END, - MARKER_TYPE_BENCHMARK_START, - setupCore, - teardownCore, -} from "@codspeed/core"; -import { Bench, Fn, Task } from "tinybench"; +import { setupCore, teardownCore } from "@codspeed/core"; +import { Bench, Task } from "tinybench"; import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; @@ -56,57 +49,12 @@ export abstract class BaseBenchRunner { return this.bench.tasks; } - protected wrapWithInstrumentHooks(fn: () => T, uri: string): T { - const runStart = this.openInstrumentWindow(); - try { - return fn(); - } finally { - this.closeInstrumentWindow(uri, runStart); - } - } - - protected async wrapWithInstrumentHooksAsync( - fn: Fn, - uri: string, - ): Promise { - const runStart = this.openInstrumentWindow(); - try { - return await fn(); - } finally { - this.closeInstrumentWindow(uri, runStart); - } - } - - protected openInstrumentWindow(): bigint { - InstrumentHooks.startBenchmark(); - return InstrumentHooks.currentTimestamp(); - } - - protected closeInstrumentWindow(uri: string, runStart: bigint): void { - const runEnd = InstrumentHooks.currentTimestamp(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - this.sendBenchmarkMarkers(runStart, runEnd); - } - protected abstract getModeName(): string; protected abstract runTaskAsync(task: Task, uri: string): Promise; protected abstract runTaskSync(task: Task, uri: string): void; protected abstract finalizeAsyncRun(): Task[]; protected abstract finalizeSyncRun(): Task[]; - private sendBenchmarkMarkers(runStart: bigint, runEnd: bigint): void { - if (getInstrumentMode() !== "walltime") { - return; - } - InstrumentHooks.addMarker( - process.pid, - MARKER_TYPE_BENCHMARK_START, - runStart, - ); - InstrumentHooks.addMarker(process.pid, MARKER_TYPE_BENCHMARK_END, runEnd); - } - public setupBenchMethods(): void { this.bench.run = async () => { this.setupBenchRun(); diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 4963fc26..90e6b700 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -1,5 +1,8 @@ import { calculateQuantiles, + InstrumentHooks, + MARKER_TYPE_BENCHMARK_END, + MARKER_TYPE_BENCHMARK_START, mongoMeasurement, msToNs, msToS, @@ -54,20 +57,39 @@ class WalltimeBenchRunner extends BaseBenchRunner { opts.setup = (task, mode) => { const setupResult = userSetup(task, mode); if (mode === "run") { - this.runStart = this.openInstrumentWindow(); + InstrumentHooks.startBenchmark(); + this.runStart = InstrumentHooks.currentTimestamp(); } return setupResult; }; opts.teardown = (task, mode) => { if (mode === "run" && task) { - this.closeInstrumentWindow(this.getTaskUri(task), this.runStart!); + const runEnd = InstrumentHooks.currentTimestamp(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark( + process.pid, + this.getTaskUri(task), + ); + if (this.runStart !== null) { + this.sendBenchmarkMarkers(this.runStart, runEnd); + } + this.runStart = null; } return userTeardown(task, mode); }; } + private sendBenchmarkMarkers(runStart: bigint, runEnd: bigint): void { + InstrumentHooks.addMarker( + process.pid, + MARKER_TYPE_BENCHMARK_START, + runStart, + ); + InstrumentHooks.addMarker(process.pid, MARKER_TYPE_BENCHMARK_END, runEnd); + } + protected async runTaskAsync(task: Task, uri: string): Promise { // run the warmup of the task right before its actual run if (getBenchOptions(this.bench).warmup) {