From 079e488bfcf6034aeb3f13e4ad1e706db60de4d8 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 3 Jun 2026 14:35:24 +0200 Subject: [PATCH 1/4] chore: bump instrument-hooks --- packages/core/src/native_core/instruments/hooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/native_core/instruments/hooks b/packages/core/src/native_core/instruments/hooks index d83209f9..b9ddb5bc 160000 --- a/packages/core/src/native_core/instruments/hooks +++ b/packages/core/src/native_core/instruments/hooks @@ -1 +1 @@ -Subproject commit d83209f91683cf4c1677bdde28e0e43ca201f6ed +Subproject commit b9ddb5bc654b2e6fa13eb18efcd3a45e7ecda0bb From 6ad82ca49d5d22209c36fdd3beb42abd204d0b2f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 3 Jun 2026 14:36:02 +0200 Subject: [PATCH 2/4] chore(vitest): emit benchmark markers --- packages/vitest-plugin/src/walltime/index.ts | 30 +++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/packages/vitest-plugin/src/walltime/index.ts b/packages/vitest-plugin/src/walltime/index.ts index 0d068941..ef5a6bf7 100644 --- a/packages/vitest-plugin/src/walltime/index.ts +++ b/packages/vitest-plugin/src/walltime/index.ts @@ -1,5 +1,7 @@ import { InstrumentHooks, + MARKER_TYPE_BENCHMARK_END, + MARKER_TYPE_BENCHMARK_START, setupCore, writeWalltimeResults, } from "@codspeed/core"; @@ -66,6 +68,7 @@ export class WalltimeRunner extends NodeBenchmarkRunner { this.isTinybenchHookedWithCodspeed = true; const originalRun = tinybench.Task.prototype.run; + const pid = process.pid; const getSuiteUri = (): string => { if (this.currentSuiteId === null) { @@ -78,10 +81,29 @@ export class WalltimeRunner extends NodeBenchmarkRunner { const { fn } = this as { fn: Fn }; const suiteUri = getSuiteUri(); - function __codspeed_root_frame__() { - return fn(); + const finishRound = (roundStart: bigint): void => { + const roundEnd = InstrumentHooks.currentTimestamp(); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, roundStart); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, roundEnd); + }; + + function __codspeed_root_frame__(): unknown { + const roundStart = InstrumentHooks.currentTimestamp(); + const result = fn(); + if ( + result !== null && + typeof result === "object" && + typeof (result as PromiseLike).then === "function" + ) { + return (result as PromiseLike).then((value) => { + finishRound(roundStart); + return value; + }); + } + finishRound(roundStart); + return result; } - (this as { fn: Fn }).fn = __codspeed_root_frame__; + (this as { fn: Fn }).fn = __codspeed_root_frame__ as Fn; InstrumentHooks.startBenchmark(); await originalRun.call(this); @@ -89,7 +111,7 @@ export class WalltimeRunner extends NodeBenchmarkRunner { // Look up the URI by task name const uri = `${suiteUri}::${this.name}`; - InstrumentHooks.setExecutedBenchmark(process.pid, uri); + InstrumentHooks.setExecutedBenchmark(pid, uri); return this; }; From 374a563f7289491923ba21ad51da257594733e8e Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 2 Jun 2026 11:57:30 +0200 Subject: [PATCH 3/4] feat: add macOS-only vitest benchmark and CI job Add benches/macos.bench.ts guarded with describe.skipIf(!darwin) so it only runs on macOS, and wire the walltime-macos-test CI job to execute it via a direct `node vitest.mjs bench --run macos` invocation. --- .github/workflows/ci.yml | 27 ------------------ .github/workflows/codspeed.yml | 28 +++++++++++++++++++ packages/vitest-plugin/benches/macos.bench.ts | 16 +++++++++++ 3 files changed, 44 insertions(+), 27 deletions(-) create mode 100644 packages/vitest-plugin/benches/macos.bench.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db3745bb..f87920b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,30 +87,3 @@ jobs: env: CODSPEED_SKIP_UPLOAD: true CODSPEED_DEBUG: true - - walltime-macos-test: - runs-on: macos-latest - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 0 - submodules: true - - uses: pnpm/action-setup@v2 - - uses: actions/setup-node@v3 - with: - cache: pnpm - node-version-file: .nvmrc - - name: Restore turbo cache - uses: ./.github/actions/turbo-cache - - run: pnpm install --frozen-lockfile --prefer-offline - - run: pnpm turbo run build - - - name: Run benchmarks - uses: CodSpeedHQ/action@main - env: - CODSPEED_SKIP_UPLOAD: "true" - # Samply fails to profile pnpm targets for now - CODSPEED_PROFILER_ENABLED: "false" - with: - run: pnpm turbo run bench --filter=@codspeed/vitest-plugin - mode: walltime diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index b28c2645..be7795df 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -91,6 +91,34 @@ jobs: pnpm --workspace-concurrency 1 -r bench-tinybench pnpm --workspace-concurrency 1 -r bench-vitest + codspeed-walltime-macos: + runs-on: macos-latest + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 0 + submodules: true + - uses: pnpm/action-setup@v2 + - uses: actions/setup-node@v3 + with: + cache: pnpm + node-version-file: .nvmrc + - name: Restore turbo cache + uses: ./.github/actions/turbo-cache + - run: pnpm install --frozen-lockfile --prefer-offline + - run: pnpm turbo run build + + - name: Run macOS-only benchmarks + uses: CodSpeedHQ/action@main + with: + working-directory: packages/vitest-plugin + # Only run the macOS-only bench file: the rest of the suite already + # runs on the linux walltime job, and uploading the same benchmark + # twice for one commit is not supported. + run: pnpm turbo run bench --env-mode=loose --filter=@codspeed/vitest-plugin -- macos + mode: walltime + runner-version: branch:sip-resign-exec-redirect + electron-e2e: name: Run electron inbox e2e runs-on: codspeed-macro diff --git a/packages/vitest-plugin/benches/macos.bench.ts b/packages/vitest-plugin/benches/macos.bench.ts new file mode 100644 index 00000000..b1a9c276 --- /dev/null +++ b/packages/vitest-plugin/benches/macos.bench.ts @@ -0,0 +1,16 @@ +import { bench, describe } from "vitest"; + +const isMacOS = process.platform === "darwin"; + +function fibo(n: number): number { + if (n < 2) return 1; + return fibo(n - 1) + fibo(n - 2); +} + +// macOS-only benchmark: skipped on every other platform, so it only runs on +// the `walltime-macos-test` CI job (see .github/workflows/ci.yml). +describe.skipIf(!isMacOS)("macos only", () => { + bench("fibo darwin", () => { + fibo(30); + }); +}); From 93cf800c8c3cd71a46df5f86c61a443384040058 Mon Sep 17 00:00:00 2001 From: Matthias Heiden Date: Thu, 4 Jun 2026 18:01:49 +0200 Subject: [PATCH 4/4] chore(tinybench): emit benchmark markers --- packages/tinybench-plugin/src/shared.ts | 27 +++++++++++++++- .../tests/index.integ.test.ts | 2 ++ packages/vitest-plugin/src/walltime/index.ts | 32 ++++--------------- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index e5eeed1c..3e579f1e 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,4 +1,11 @@ -import { InstrumentHooks, setupCore, teardownCore } from "@codspeed/core"; +import { + getInstrumentMode, + InstrumentHooks, + MARKER_TYPE_BENCHMARK_END, + MARKER_TYPE_BENCHMARK_START, + setupCore, + teardownCore, +} from "@codspeed/core"; import { Bench, Fn, Task } from "tinybench"; import { getTaskUri } from "./uri"; @@ -63,14 +70,29 @@ export abstract class BaseBenchRunner { protected abstract finalizeAsyncRun(): Task[]; protected abstract finalizeSyncRun(): Task[]; + private sendRunMarkers(runStart: bigint, runEnd: bigint): void { + if (getInstrumentMode() !== "walltime") { + return; + } + InstrumentHooks.addMarker( + process.pid, + MARKER_TYPE_BENCHMARK_START, + runStart, + ); + InstrumentHooks.addMarker(process.pid, MARKER_TYPE_BENCHMARK_END, runEnd); + } + public setupBenchMethods(): void { this.bench.run = async () => { this.setupBenchRun(); + const runStart = InstrumentHooks.currentTimestamp(); for (const task of this.bench.tasks) { const uri = this.getTaskUri(task); await this.runTaskAsync(task, uri); } + const runEnd = InstrumentHooks.currentTimestamp(); + this.sendRunMarkers(runStart, runEnd); return this.finalizeAsyncRun(); }; @@ -78,10 +100,13 @@ export abstract class BaseBenchRunner { this.bench.runSync = () => { this.setupBenchRun(); + const runStart = InstrumentHooks.currentTimestamp(); for (const task of this.bench.tasks) { const uri = this.getTaskUri(task); this.runTaskSync(task, uri); } + const runEnd = InstrumentHooks.currentTimestamp(); + this.sendRunMarkers(runStart, runEnd); return this.finalizeSyncRun(); }; diff --git a/packages/tinybench-plugin/tests/index.integ.test.ts b/packages/tinybench-plugin/tests/index.integ.test.ts index 4620c6c6..75e9907f 100644 --- a/packages/tinybench-plugin/tests/index.integ.test.ts +++ b/packages/tinybench-plugin/tests/index.integ.test.ts @@ -16,6 +16,8 @@ const mockCore = vi.hoisted(() => { startBenchmark: vi.fn(), stopBenchmark: vi.fn(), setExecutedBenchmark: vi.fn(), + currentTimestamp: vi.fn().mockReturnValue(0n), + addMarker: vi.fn(), }, optimizeFunction: vi .fn() diff --git a/packages/vitest-plugin/src/walltime/index.ts b/packages/vitest-plugin/src/walltime/index.ts index ef5a6bf7..fd2c8124 100644 --- a/packages/vitest-plugin/src/walltime/index.ts +++ b/packages/vitest-plugin/src/walltime/index.ts @@ -5,7 +5,6 @@ import { setupCore, writeWalltimeResults, } from "@codspeed/core"; -import { Fn } from "tinybench"; import { RunnerTaskEventPack, RunnerTaskResultPack, @@ -78,37 +77,18 @@ export class WalltimeRunner extends NodeBenchmarkRunner { }; tinybench.Task.prototype.run = async function () { - const { fn } = this as { fn: Fn }; const suiteUri = getSuiteUri(); - const finishRound = (roundStart: bigint): void => { - const roundEnd = InstrumentHooks.currentTimestamp(); - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, roundStart); - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, roundEnd); - }; - - function __codspeed_root_frame__(): unknown { - const roundStart = InstrumentHooks.currentTimestamp(); - const result = fn(); - if ( - result !== null && - typeof result === "object" && - typeof (result as PromiseLike).then === "function" - ) { - return (result as PromiseLike).then((value) => { - finishRound(roundStart); - return value; - }); - } - finishRound(roundStart); - return result; - } - (this as { fn: Fn }).fn = __codspeed_root_frame__ as Fn; - InstrumentHooks.startBenchmark(); + const runStart = InstrumentHooks.currentTimestamp(); await originalRun.call(this); + const runEnd = InstrumentHooks.currentTimestamp(); InstrumentHooks.stopBenchmark(); + // Emit a single marker pair covering the whole measurement run + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, runStart); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); + // Look up the URI by task name const uri = `${suiteUri}::${this.name}`; InstrumentHooks.setExecutedBenchmark(pid, uri);