diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f87920b1..b6133882 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,12 @@ jobs: matrix: node-version: ["18", "20.5.1"] example: ${{ fromJson(needs.list-examples.outputs.examples) }} + # tinybench v5+ requires Node >=20, so its examples cannot run on Node 18 + exclude: + - node-version: "18" + example: "with-tinybench-v5" + - node-version: "18" + example: "with-tinybench-v6" fail-fast: false steps: - uses: "actions/checkout@v4" diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index d23925b5..e57462ec 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -82,8 +82,12 @@ jobs: - name: Run benchmarks # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` uses: CodSpeedHQ/action@main + env: + CODSPEED_WALLTIME_PROFILER: samply with: mode: walltime + # TODO: Remove this once runner is released + runner-version: v4.17.7-alpha.2 run: | pnpm turbo run bench --filter=@codspeed/tinybench-plugin pnpm turbo run bench --filter=@codspeed/vitest-plugin @@ -173,3 +177,4 @@ jobs: working-directory: examples/with-electron-and-walltime run: xvfb-run -a pnpm bench:electron mode: walltime + runner-version: v4.17.7-alpha.2 diff --git a/CLAUDE.md b/CLAUDE.md index 496e1bd9..b82671a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -88,4 +88,10 @@ Based on the codebase analysis, to add stats access features: ## Repository Management Memories - Use pnpm instead of npm -- To run tests in a package use moon :test \ No newline at end of file +- The monorepo uses [Turborepo](https://turborepo.com). Run a task for a single + package with a filter, using the package's full name (e.g. `@codspeed/tinybench-plugin`): + - Test: `pnpm turbo run test --filter=` + - Build: `pnpm turbo run build --filter=` + - Typecheck: `pnpm turbo run typecheck --filter=` + - Lint: `pnpm turbo run lint --filter=` + - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`). \ No newline at end of file diff --git a/examples/with-tinybench-v5/package.json b/examples/with-tinybench-v5/package.json new file mode 100644 index 00000000..98a132cb --- /dev/null +++ b/examples/with-tinybench-v5/package.json @@ -0,0 +1,15 @@ +{ + "name": "with-tinybench-v5", + "private": true, + "type": "module", + "scripts": { + "bench-tinybench": "node --loader esbuild-register/loader -r esbuild-register tinybench.ts", + "typecheck": "tsc --noEmit --pretty" + }, + "devDependencies": { + "@codspeed/tinybench-plugin": "workspace:*", + "esbuild-register": "^3.4.2", + "tinybench": "^5.1.0", + "typescript": "^5.1.3" + } +} diff --git a/examples/with-tinybench-v5/tinybench.ts b/examples/with-tinybench-v5/tinybench.ts new file mode 100644 index 00000000..f2788bd0 --- /dev/null +++ b/examples/with-tinybench-v5/tinybench.ts @@ -0,0 +1,39 @@ +import { withCodSpeed } from "@codspeed/tinybench-plugin"; +import { Bench } from "tinybench"; + +function recursiveFibonacci(n: number): number { + if (n < 2) { + return n; + } + return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); +} + +// The plugin resolves tinybench's `Bench` type from the version hoisted at the +// monorepo root, which differs from the major pinned in this example. A real +// consumer has a single installed tinybench, so this mismatch is local to the +// workspace; `@ts-expect-error` keeps the example typechecking and flags us if +// the discrepancy ever resolves on its own. +// @ts-expect-error cross-version Bench type mismatch within the monorepo +const bench = withCodSpeed(new Bench({ time: 100, warmup: true })); + +bench + .add("recursive fibo 10", () => { + recursiveFibonacci(10); + }) + // Register the per-task hooks so the plugin's handling of them is exercised. + .add( + "recursive fibo 15 with hooks", + () => { + recursiveFibonacci(15); + }, + { + beforeAll() {}, + beforeEach() {}, + afterEach() {}, + afterAll() {}, + }, + ); + +bench.run().then(() => { + console.table(bench.table()); +}); diff --git a/examples/with-tinybench-v5/tsconfig.json b/examples/with-tinybench-v5/tsconfig.json new file mode 100644 index 00000000..7b8550e2 --- /dev/null +++ b/examples/with-tinybench-v5/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/examples/with-tinybench-v6/package.json b/examples/with-tinybench-v6/package.json new file mode 100644 index 00000000..0fe8d728 --- /dev/null +++ b/examples/with-tinybench-v6/package.json @@ -0,0 +1,15 @@ +{ + "name": "with-tinybench-v6", + "private": true, + "type": "module", + "scripts": { + "bench-tinybench": "node --loader esbuild-register/loader -r esbuild-register tinybench.ts", + "typecheck": "tsc --noEmit --pretty" + }, + "devDependencies": { + "@codspeed/tinybench-plugin": "workspace:*", + "esbuild-register": "^3.4.2", + "tinybench": "^6.0.2", + "typescript": "^5.1.3" + } +} diff --git a/examples/with-tinybench-v6/tinybench.ts b/examples/with-tinybench-v6/tinybench.ts new file mode 100644 index 00000000..f2788bd0 --- /dev/null +++ b/examples/with-tinybench-v6/tinybench.ts @@ -0,0 +1,39 @@ +import { withCodSpeed } from "@codspeed/tinybench-plugin"; +import { Bench } from "tinybench"; + +function recursiveFibonacci(n: number): number { + if (n < 2) { + return n; + } + return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); +} + +// The plugin resolves tinybench's `Bench` type from the version hoisted at the +// monorepo root, which differs from the major pinned in this example. A real +// consumer has a single installed tinybench, so this mismatch is local to the +// workspace; `@ts-expect-error` keeps the example typechecking and flags us if +// the discrepancy ever resolves on its own. +// @ts-expect-error cross-version Bench type mismatch within the monorepo +const bench = withCodSpeed(new Bench({ time: 100, warmup: true })); + +bench + .add("recursive fibo 10", () => { + recursiveFibonacci(10); + }) + // Register the per-task hooks so the plugin's handling of them is exercised. + .add( + "recursive fibo 15 with hooks", + () => { + recursiveFibonacci(15); + }, + { + beforeAll() {}, + beforeEach() {}, + afterEach() {}, + afterAll() {}, + }, + ); + +bench.run().then(() => { + console.table(bench.table()); +}); diff --git a/examples/with-tinybench-v6/tsconfig.json b/examples/with-tinybench-v6/tsconfig.json new file mode 100644 index 00000000..7b8550e2 --- /dev/null +++ b/examples/with-tinybench-v6/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2fd780b5..1f23273f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -56,7 +56,10 @@ export const setupCore = () => { } native_core.InstrumentHooks.setIntegration("codspeed-node", __VERSION__); - linuxPerf.start(); + // In walltime, we use node's native profiling options rather than our own, cf `getV8Flags` + if (getCodspeedRunnerMode() !== "walltime") { + linuxPerf.start(); + } checkV8Flags(); // Collect Node.js runtime environment to detect changes that could @@ -68,7 +71,9 @@ export const setupCore = () => { }; export const teardownCore = () => { - linuxPerf.stop(); + if (getCodspeedRunnerMode() !== "walltime") { + linuxPerf.stop(); + } }; export type { diff --git a/packages/core/src/introspection.ts b/packages/core/src/introspection.ts index 77dcfed0..5bb232ec 100644 --- a/packages/core/src/introspection.ts +++ b/packages/core/src/introspection.ts @@ -1,32 +1,56 @@ import { writeFileSync } from "fs"; +import path from "path"; + import { getInstrumentMode } from "."; const CUSTOM_INTROSPECTION_EXIT_CODE = 0; +const V8_LOG_FILENAME_PATTERN = "codspeed-v8-%p.log"; + export const getV8Flags = () => { const nodeVersionMajor = parseInt(process.version.slice(1).split(".")[0]); const instrumentMode = getInstrumentMode(); const flags = ["--interpreted-frames-native-stack", "--allow-natives-syntax"]; - if (instrumentMode === "analysis") { - flags.push( - ...[ - "--hash-seed=1", - "--random-seed=1", - "--no-opt", - "--predictable", - "--predictable-gc-schedule", - "--expose-gc", - "--no-concurrent-sweeping", - "--max-old-space-size=4096", - ], - ); - if (nodeVersionMajor < 18) { - flags.push("--no-randomize-hashes"); + switch (instrumentMode) { + case "analysis": { + flags.push( + ...[ + "--hash-seed=1", + "--random-seed=1", + "--no-opt", + "--predictable", + "--predictable-gc-schedule", + "--expose-gc", + "--no-concurrent-sweeping", + "--max-old-space-size=4096", + ], + ); + if (nodeVersionMajor < 18) { + flags.push("--no-randomize-hashes"); + } + if (nodeVersionMajor < 20) { + flags.push("--no-scavenge-task"); + } + + break; } - if (nodeVersionMajor < 20) { - flags.push("--no-scavenge-task"); + + case "walltime": { + // Emit the V8 jitdump + flags.push("--perf-prof"); + const v8LogDir = process.env.CODSPEED_V8_LOG; + if (v8LogDir) { + flags.push( + ...[ + "--log-code", + "--no-log-source-code", + "--no-logfile-per-isolate", + `--logfile=${path.join(v8LogDir, V8_LOG_FILENAME_PATTERN)}`, + ], + ); + } } } diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index bdaca56b..65bd6183 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -3,10 +3,11 @@ import { InstrumentHooks, mongoMeasurement, optimizeFunction, + optimizeFunctionSync, wrapWithRootFrame, wrapWithRootFrameSync, } from "@codspeed/core"; -import { Bench, Fn, FnOptions, Task } from "tinybench"; +import { Bench, Task } from "tinybench"; import { BaseBenchRunner } from "./shared"; export function setupCodspeedAnalysisBench( @@ -28,7 +29,7 @@ class AnalysisBenchRunner extends BaseBenchRunner { } protected async runTaskAsync(task: Task, uri: string): Promise { - const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn }; + const { fnOpts, fn } = this.getTaskData(task); await fnOpts?.beforeAll?.call(task, "run"); await optimizeFunction(async () => { @@ -40,7 +41,14 @@ class AnalysisBenchRunner extends BaseBenchRunner { await mongoMeasurement.start(uri); global.gc?.(); - await this.wrapWithInstrumentHooksAsync(wrapWithRootFrame(fn), uri); + const rootFrame = wrapWithRootFrame(fn); + InstrumentHooks.startBenchmark(); + try { + await rootFrame(); + } finally { + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + } await mongoMeasurement.stop(uri); await fnOpts?.afterEach?.call(task, "run"); @@ -50,12 +58,27 @@ class AnalysisBenchRunner extends BaseBenchRunner { } protected runTaskSync(task: Task, uri: string): void { - const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn }; + const { fnOpts, fn } = this.getTaskData(task); fnOpts?.beforeAll?.call(task, "run"); + + optimizeFunctionSync(async () => { + fnOpts?.beforeEach?.call(task, "run"); + await fn(); + fnOpts?.afterEach?.call(task, "run"); + }); + fnOpts?.beforeEach?.call(task, "run"); - this.wrapWithInstrumentHooks(wrapWithRootFrameSync(fn), uri); + global.gc?.(); + const rootFrame = wrapWithRootFrameSync(fn); + InstrumentHooks.startBenchmark(); + try { + rootFrame(); + } finally { + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + } fnOpts?.afterEach?.call(task, "run"); fnOpts?.afterAll?.call(task, "run"); diff --git a/packages/tinybench-plugin/src/benchOptions.ts b/packages/tinybench-plugin/src/benchOptions.ts new file mode 100644 index 00000000..a1ffef42 --- /dev/null +++ b/packages/tinybench-plugin/src/benchOptions.ts @@ -0,0 +1,24 @@ +import { Bench, Hook } from "tinybench"; + +// The benchmark options tinybench resolves: timing knobs plus the suite-level +// setup/teardown hooks. tinybench guarantees `setup`/`teardown` are populated +// (at least with no-op defaults), so they are non-optional here. +export interface ResolvedBenchOptions { + setup: Hook; + teardown: Hook; + warmup?: boolean; + warmupIterations?: number; + warmupTime?: number; + iterations?: number; + time?: number; + retainSamples?: boolean; +} + +// Up to tinybench v5 the resolved options lived under `bench.opts`; from v6 they +// are flattened onto the bench instance itself. Returning the holder object +// (rather than a copy) keeps in-place mutation of `setup`/`teardown` working, +// which the walltime runner relies on to bracket its instrumentation window. +export function getBenchOptions(bench: Bench): ResolvedBenchOptions { + const opts = (bench as unknown as { opts?: ResolvedBenchOptions }).opts; + return opts ?? (bench as unknown as ResolvedBenchOptions); +} diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index 64be2d8d..298f97b9 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -7,9 +7,11 @@ import { SetupInstrumentsRequestBody, SetupInstrumentsResponse, tryIntrospect, + wrapWithRootFrameSync, } from "@codspeed/core"; import { Bench } from "tinybench"; import { setupCodspeedAnalysisBench } from "./analysis"; +import { getOrCreateTaskDataMap } from "./taskData"; import { getOrCreateUriMap } from "./uri"; import { setupCodspeedWalltimeBench } from "./walltime"; @@ -22,9 +24,11 @@ export function withCodSpeed(bench: Bench): Bench { } const rootCallingFile = getCallingFile(1); + const instrumentMode = getInstrumentMode(); // Compute and register URI for bench const uriMap = getOrCreateUriMap(bench); + const taskDataMap = getOrCreateTaskDataMap(bench); const rawAdd = bench.add; bench.add = (name, fn, opts?) => { const callingFile = getCallingFile(1); @@ -34,10 +38,17 @@ export function withCodSpeed(bench: Bench): Bench { } uri += `::${name}`; uriMap.set(name, uri); - return rawAdd.bind(bench)(name, fn, opts); + taskDataMap.set(name, { fn, fnOpts: opts }); + // In walltime mode the task is driven by tinybench's own measured loop, so + // the root frame must be baked into the function tinybench stores rather + // than injected later (the function is an inaccessible private field on the + // task from tinybench v6 onwards). The sync wrapper is transparent to the + // function's return value, so it preserves tinybench's sync/async handling. + const registeredFn = + instrumentMode === "walltime" ? wrapWithRootFrameSync(fn) : fn; + return rawAdd.bind(bench)(name, registeredFn, opts); }; - const instrumentMode = getInstrumentMode(); if (instrumentMode === "analysis") { setupCodspeedAnalysisBench(bench, rootCallingFile); } else if (instrumentMode === "walltime") { diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 093ed0ec..0fb811c5 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,12 +1,6 @@ -import { - getInstrumentMode, - InstrumentHooks, - MARKER_TYPE_BENCHMARK_END, - MARKER_TYPE_BENCHMARK_START, - setupCore, - teardownCore, -} from "@codspeed/core"; -import { Bench, Fn, Task } from "tinybench"; +import { setupCore, teardownCore } from "@codspeed/core"; +import { Bench, Task } from "tinybench"; +import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; declare const __VERSION__: string; @@ -35,6 +29,16 @@ export abstract class BaseBenchRunner { return getTaskUri(this.bench, task.name, this.rootCallingFile); } + protected getTaskData(task: Task): CapturedTaskData { + const data = getTaskData(this.bench, task.name); + if (!data) { + throw new Error( + `[CodSpeed] No captured function found for task "${task.name}"`, + ); + } + return data; + } + protected logTaskCompletion(uri: string, status: string): void { console.log(`[CodSpeed] ${status} ${uri}`); } @@ -45,53 +49,12 @@ export abstract class BaseBenchRunner { return this.bench.tasks; } - protected wrapWithInstrumentHooks(fn: () => T, uri: string): T { - InstrumentHooks.startBenchmark(); - const runStart = InstrumentHooks.currentTimestamp(); - try { - return fn(); - } finally { - const runEnd = InstrumentHooks.currentTimestamp(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - this.sendBenchmarkMarkers(runStart, runEnd); - } - } - - protected async wrapWithInstrumentHooksAsync( - fn: Fn, - uri: string, - ): Promise { - InstrumentHooks.startBenchmark(); - const runStart = InstrumentHooks.currentTimestamp(); - try { - return await fn(); - } finally { - const runEnd = InstrumentHooks.currentTimestamp(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - this.sendBenchmarkMarkers(runStart, runEnd); - } - } - protected abstract getModeName(): string; protected abstract runTaskAsync(task: Task, uri: string): Promise; protected abstract runTaskSync(task: Task, uri: string): void; protected abstract finalizeAsyncRun(): Task[]; protected abstract finalizeSyncRun(): Task[]; - private sendBenchmarkMarkers(runStart: bigint, runEnd: bigint): void { - if (getInstrumentMode() !== "walltime") { - return; - } - InstrumentHooks.addMarker( - process.pid, - MARKER_TYPE_BENCHMARK_START, - runStart, - ); - InstrumentHooks.addMarker(process.pid, MARKER_TYPE_BENCHMARK_END, runEnd); - } - public setupBenchMethods(): void { this.bench.run = async () => { this.setupBenchRun(); diff --git a/packages/tinybench-plugin/src/taskData.ts b/packages/tinybench-plugin/src/taskData.ts new file mode 100644 index 00000000..a5ecb2ab --- /dev/null +++ b/packages/tinybench-plugin/src/taskData.ts @@ -0,0 +1,30 @@ +import { Bench, Fn, FnOptions } from "tinybench"; + +// tinybench stores the benchmark function and its options as private fields on +// `Task`. They were reachable through casts on older majors but became true +// `#private` fields in v6, so we capture them ourselves when `bench.add` runs +// and key them by task name, mirroring the URI map. +export interface CapturedTaskData { + fn: Fn; + fnOpts?: FnOptions; +} + +const taskDataMap = new WeakMap>(); + +export function getOrCreateTaskDataMap( + bench: Bench, +): Map { + let map = taskDataMap.get(bench); + if (!map) { + map = new Map(); + taskDataMap.set(bench, map); + } + return map; +} + +export function getTaskData( + bench: Bench, + taskName: string, +): CapturedTaskData | undefined { + return taskDataMap.get(bench)?.get(taskName); +} diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 9944a61a..90e6b700 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -1,15 +1,17 @@ import { calculateQuantiles, + InstrumentHooks, + MARKER_TYPE_BENCHMARK_END, + MARKER_TYPE_BENCHMARK_START, mongoMeasurement, msToNs, msToS, - wrapWithRootFrame, - wrapWithRootFrameSync, writeWalltimeResults, type BenchmarkStats, type Benchmark as CodspeedBenchmark, } from "@codspeed/core"; -import { Bench, Fn, Task, TaskResult } from "tinybench"; +import { Bench, Task, TaskResult } from "tinybench"; +import { getBenchOptions } from "./benchOptions"; import { BaseBenchRunner } from "./shared"; export function setupCodspeedWalltimeBench( @@ -17,41 +19,97 @@ export function setupCodspeedWalltimeBench( rootCallingFile: string, ): void { const runner = new WalltimeBenchRunner(bench, rootCallingFile); + runner.installInstrumentHooks(); runner.setupBenchMethods(); } class WalltimeBenchRunner extends BaseBenchRunner { private codspeedBenchmarks: CodspeedBenchmark[] = []; + // Carries the window start timestamp from the setup hook to the teardown + // hook. Tasks run strictly sequentially, so a single field is enough. + private runStart: bigint | null = null; + protected getModeName(): string { return "walltime mode"; } - protected async runTaskAsync(task: Task, uri: string): Promise { - // Override the function under test to add a static frame - this.wrapTaskFunction(task, true); + /** + * Drive the instrumentation window from the task's setup/teardown hooks so it + * brackets only tinybench's measured loop, excluding warmup and the + * statistics computation (`processRunResult`) that surround it. The user's + * own hooks are preserved and still run in their original order relative to + * the work under test. + */ + public installInstrumentHooks(): void { + // The resolved options expose `setup`/`teardown` as typed `Readonly`, but + // tinybench populates them with (at least) no-op defaults and lets them be + // reassigned at runtime. + const opts = getBenchOptions(this.bench); + + // We build the walltime statistics from the per-round latency samples. + // tinybench stopped retaining them by default in v6, so opt back in. + opts.retainSamples = true; + + const userSetup = opts.setup; + const userTeardown = opts.teardown; + + opts.setup = (task, mode) => { + const setupResult = userSetup(task, mode); + if (mode === "run") { + InstrumentHooks.startBenchmark(); + this.runStart = InstrumentHooks.currentTimestamp(); + } + return setupResult; + }; + + opts.teardown = (task, mode) => { + if (mode === "run" && task) { + const runEnd = InstrumentHooks.currentTimestamp(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark( + process.pid, + this.getTaskUri(task), + ); + if (this.runStart !== null) { + this.sendBenchmarkMarkers(this.runStart, runEnd); + } + + this.runStart = null; + } + return userTeardown(task, mode); + }; + } + + private sendBenchmarkMarkers(runStart: bigint, runEnd: bigint): void { + InstrumentHooks.addMarker( + process.pid, + MARKER_TYPE_BENCHMARK_START, + runStart, + ); + InstrumentHooks.addMarker(process.pid, MARKER_TYPE_BENCHMARK_END, runEnd); + } + protected async runTaskAsync(task: Task, uri: string): Promise { // run the warmup of the task right before its actual run - if (this.bench.opts.warmup) { + if (getBenchOptions(this.bench).warmup) { await task.warmup(); } await mongoMeasurement.start(uri); - await this.wrapWithInstrumentHooksAsync(() => task.run(), uri); + await task.run(); await mongoMeasurement.stop(uri); this.registerCodspeedBenchmarkFromTask(task); } - protected runTaskSync(task: Task, uri: string): void { - // Override the function under test to add a static frame - this.wrapTaskFunction(task, false); - - if (this.bench.opts.warmup) { - task.warmup(); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + protected runTaskSync(task: Task, _uri: string): void { + if (getBenchOptions(this.bench).warmup) { + task.warmupSync(); } - this.wrapWithInstrumentHooks(() => task.runSync(), uri); + task.runSync(); this.registerCodspeedBenchmarkFromTask(task); } @@ -64,14 +122,6 @@ class WalltimeBenchRunner extends BaseBenchRunner { return this.finalizeWalltimeRun(false); } - private wrapTaskFunction(task: Task, isAsync: boolean): void { - const { fn } = task as unknown as { fn: Fn }; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (task as any).fn = isAsync - ? wrapWithRootFrame(fn) - : wrapWithRootFrameSync(fn); - } - private registerCodspeedBenchmarkFromTask(task: Task): void { const uri = this.getTaskUri(task); @@ -80,8 +130,9 @@ class WalltimeBenchRunner extends BaseBenchRunner { return; } - const warmupIterations = this.bench.opts.warmup - ? (this.bench.opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT) + const opts = getBenchOptions(this.bench); + const warmupIterations = opts.warmup + ? (opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT) : 0; const stats = convertTinybenchResultToBenchmarkStats( task.result, @@ -92,13 +143,11 @@ class WalltimeBenchRunner extends BaseBenchRunner { name: task.name, uri, config: { - max_rounds: this.bench.opts.iterations ?? null, - max_time_ns: this.bench.opts.time ? msToNs(this.bench.opts.time) : null, + max_rounds: opts.iterations ?? null, + max_time_ns: opts.time ? msToNs(opts.time) : null, min_round_time_ns: null, // tinybench does not have an option for this warmup_time_ns: - this.bench.opts.warmup && this.bench.opts.warmupTime - ? msToNs(this.bench.opts.warmupTime) - : null, + opts.warmup && opts.warmupTime ? msToNs(opts.warmupTime) : null, }, stats, }); diff --git a/packages/vitest-plugin/src/walltime/index.ts b/packages/vitest-plugin/src/walltime/index.ts index 92a09916..18c71215 100644 --- a/packages/vitest-plugin/src/walltime/index.ts +++ b/packages/vitest-plugin/src/walltime/index.ts @@ -6,6 +6,7 @@ import { wrapWithRootFrame, writeWalltimeResults, } from "@codspeed/core"; +import type * as tinybench from "tinybench"; import { RunnerTaskEventPack, RunnerTaskResultPack, @@ -15,6 +16,27 @@ import { NodeBenchmarkRunner } from "vitest/runners"; import { patchRootSuiteWithFullFilePath } from "../common"; import { extractBenchmarkResults } from "./utils"; +type Tinybench = typeof tinybench; + +/** A tinybench task, exposing the `fn` the runner wraps with the root frame. */ +interface TinybenchTask { + name: string; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fn: (...args: any[]) => any; +} + +/** tinybench's per-task setup/teardown hook signature. */ +type TinybenchHook = ( + task: TinybenchTask, + mode: "run" | "warmup", +) => Promise | void; + +/** The mutable subset of a tinybench Bench the runner reaches into. */ +interface TinybenchBench { + setup: TinybenchHook; + teardown: TinybenchHook; +} + /** * WalltimeRunner uses Vitest's default benchmark execution * and extracts results from the suite after completion @@ -24,6 +46,9 @@ export class WalltimeRunner extends NodeBenchmarkRunner { private suiteUris = new Map(); /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks private currentSuiteId: string | null = null; + // Carries the window start timestamp from the setup hook to the teardown + // hook. Tasks run strictly sequentially, so a single field is enough. + private runStart: bigint | null = null; async runSuite(suite: RunnerTestSuite): Promise { patchRootSuiteWithFullFilePath(suite); @@ -59,59 +84,118 @@ export class WalltimeRunner extends NodeBenchmarkRunner { } } - async importTinybench() { + private getBenchmarkUri(taskName: string): string { + if (this.currentSuiteId === null) { + throw new Error("currentSuiteId is null - something went wrong"); + } + const suiteUri = this.suiteUris.get(this.currentSuiteId) || ""; + return `${suiteUri}::${taskName}`; + } + + async importTinybench(): Promise { const tinybench = await super.importTinybench(); - if (this.isTinybenchHookedWithCodspeed) { - return tinybench; + // `tinybench` is a frozen ES module namespace, so the `Bench` export cannot + // be reassigned. Mutating the shared `Task.prototype` in place is allowed + // and only needs to happen once; the instrumented `Bench` is handed back + // through a fresh module-shaped object that Vitest destructures from. + if (!this.isTinybenchHookedWithCodspeed) { + this.isTinybenchHookedWithCodspeed = true; + this.patchTaskWithRootFrame(tinybench); } - this.isTinybenchHookedWithCodspeed = true; - const originalRun = tinybench.Task.prototype.run; - const pid = process.pid; - - const getSuiteUri = (): string => { - if (this.currentSuiteId === null) { - throw new Error("currentSuiteId is null - something went wrong"); - } - return this.suiteUris.get(this.currentSuiteId) || ""; + return { + ...tinybench, + Bench: this.createInstrumentedBench(tinybench), }; + } - tinybench.Task.prototype.run = async function () { - const suiteUri = getSuiteUri(); + /** + * Wrap each task's function with the root frame so collected stacks can be + * attributed to a benchmark. The window itself is driven by the bench's + * setup/teardown hooks (see createInstrumentedBench). + */ + private patchTaskWithRootFrame(tinybench: Tinybench): void { + const originalRun = tinybench.Task.prototype.run; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const task = this as any; + tinybench.Task.prototype.run = async function () { + const task = this as unknown as TinybenchTask; const originalFn = task.fn; task.fn = wrapWithRootFrame(() => originalFn.call(task)); - InstrumentHooks.startBenchmark(); - const runStart = InstrumentHooks.currentTimestamp(); try { await originalRun.call(this); } finally { - const runEnd = InstrumentHooks.currentTimestamp(); task.fn = originalFn; + } - // Benchmark markers must land inside the sample window opened by - // startBenchmark(), so they have to be emitted before stopBenchmark() - // closes it. The runner consumes the FIFO stream in order, so a marker - // sent after StopBenchmark falls outside the sample and breaks the - // expected SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, runStart); - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); + return this; + }; + } - InstrumentHooks.stopBenchmark(); + /** + * Drive the instrumentation window from each bench's run-mode setup/teardown + * hooks so it brackets only tinybench's measured loop, excluding the warmup + * that Vitest runs beforehand and the statistics computation tinybench + * performs after the loop. Wrapping the whole `Task.run()` would otherwise + * fold all of that framework overhead into the recorded sample. + * + * User-provided hooks are preserved and keep their order relative to the work + * under test. + */ + private createInstrumentedBench( + tinybench: Tinybench, + ): typeof tinybench.Bench { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const runner = this; + const OriginalBench = tinybench.Bench; + + class InstrumentedBench extends OriginalBench { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(...benchArgs: any[]) { + super(...benchArgs); + runner.installInstrumentHooks(this as unknown as TinybenchBench); + } + } - // Look up the URI by task name - const uri = `${suiteUri}::${this.name}`; - InstrumentHooks.setExecutedBenchmark(pid, uri); + return InstrumentedBench; + } + + private installInstrumentHooks(bench: TinybenchBench): void { + const userSetup = bench.setup; + const userTeardown = bench.teardown; + + bench.setup = async (task, mode) => { + await userSetup(task, mode); + if (mode === "run") { + InstrumentHooks.startBenchmark(); + this.runStart = InstrumentHooks.currentTimestamp(); } + }; - return this; + bench.teardown = async (task, mode) => { + if (mode === "run") { + this.closeInstrumentWindow(this.getBenchmarkUri(task.name)); + } + await userTeardown(task, mode); }; + } + + private closeInstrumentWindow(uri: string): void { + const runEnd = InstrumentHooks.currentTimestamp(); + const pid = process.pid; - return tinybench; + // Benchmark markers must land inside the sample window opened by + // startBenchmark(), so they have to be emitted before stopBenchmark() + // closes it. The runner consumes the FIFO stream in order, so a marker + // sent after StopBenchmark falls outside the sample and breaks the + // expected SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, this.runStart!); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); + + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(pid, uri); + this.runStart = null; } // Allow tinybench to retrieve the path to the currently running suite diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b2aeda91..d59ca9e2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -108,6 +108,36 @@ importers: specifier: ^4.0.1 version: 4.0.1 + examples/with-tinybench-v5: + devDependencies: + '@codspeed/tinybench-plugin': + specifier: workspace:* + version: link:../../packages/tinybench-plugin + esbuild-register: + specifier: ^3.4.2 + version: 3.4.2(esbuild@0.25.9) + tinybench: + specifier: ^5.1.0 + version: 5.1.0 + typescript: + specifier: ^5.1.3 + version: 5.8.3 + + examples/with-tinybench-v6: + devDependencies: + '@codspeed/tinybench-plugin': + specifier: workspace:* + version: link:../../packages/tinybench-plugin + esbuild-register: + specifier: ^3.4.2 + version: 3.4.2(esbuild@0.25.9) + tinybench: + specifier: ^6.0.2 + version: 6.0.2 + typescript: + specifier: ^5.1.3 + version: 5.8.3 + examples/with-typescript-cjs: devDependencies: '@codspeed/benchmark.js-plugin': @@ -5960,6 +5990,14 @@ packages: resolution: {integrity: sha512-Nb1srn7dvzkVx0J5h1vq8f48e3TIcbrS7e/UfAI/cDSef/n8yLh4zsAEsFkfpw6auTY+ZaspEvam/xs8nMnotQ==} engines: {node: '>=18.0.0'} + tinybench@5.1.0: + resolution: {integrity: sha512-LXKNtFualiKOm6gADe1UXPtf8+Nfn1CtPMEHAT33Fd2YjQatrujkDcK0+4wRC1X6t7fxUDXUs6BsvuIgfkDgDg==} + engines: {node: '>=20.0.0'} + + tinybench@6.0.2: + resolution: {integrity: sha512-FlHoQpcFvCzeXK5kVPvV7IVgW/hs/B36QWTz876iSdeJguBDfdTSRQmYmaHX+fQNt4hp+gEFB2XXw+8hT4/y8A==} + engines: {node: '>=20.0.0'} + tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} @@ -13495,6 +13533,10 @@ snapshots: tinybench@4.0.1: {} + tinybench@5.1.0: {} + + tinybench@6.0.2: {} + tinyexec@0.3.2: {} tinyexec@1.0.2: {}