fix: move outputFile to benchmark, allow --reporter for benchmark (#2037)

docs/config/index.md

@@ -168,6 +168,15 @@ When defined, Vitest will run all matched files with `import.meta.vitest` inside
 
 Custom reporter for output. Can contain one or more built-in report names, reporter instances, and/or paths to custom reporters.
 
+### benchmark.outputFile
+
+- **Type:** `string | Record<string, string>`
+
+Write benchmark results to a file when the `--reporter=json` option is also specified.
+By providing an object instead of a string you can define individual outputs when using multiple reporters.
+
+To provide object via CLI command, use the following syntax: `--outputFile.json=./path --outputFile.junit=./other-path`.
+
 ### alias
 
 - **Type:** `Record<string, string> | Array<{ find: string | RegExp, replacement: string, customResolver?: ResolverFunction | ResolverObject }>`

packages/vitest/src/defaults.ts

@@ -3,7 +3,7 @@ import type { BenchmarkUserOptions, ResolvedCoverageOptions, UserConfig } from '
 export const defaultInclude = ['**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}']
 export const defaultExclude = ['**/node_modules/**', '**/dist/**', '**/cypress/**', '**/.{idea,git,cache,output,temp}/**']
 
-export const benchmarkConfigDefaults: Required<BenchmarkUserOptions> = {
+export const benchmarkConfigDefaults: Required<Omit<BenchmarkUserOptions, 'outputFile'>> = {
   include: ['**/*.{bench,benchmark}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
   exclude: defaultExclude,
   includeSource: [],

packages/vitest/src/node/config.ts

@@ -10,6 +10,7 @@ import { toArray } from '../utils'
 import { VitestCache } from './cache'
 import { BaseSequencer } from './sequencers/BaseSequencer'
 import { RandomSequencer } from './sequencers/RandomSequencer'
+import type { BenchmarkBuiltinReporters } from './reporters'
 
 const extraInlineDeps = [
   /^(?!.*(?:node_modules)).*\.mjs$/,
@@ -169,6 +170,18 @@ export function resolveConfig(
     resolved.include = resolved.benchmark.include
     resolved.exclude = resolved.benchmark.exclude
     resolved.includeSource = resolved.benchmark.includeSource
+    const reporters = Array.from(new Set<BenchmarkBuiltinReporters>([
+      ...toArray(resolved.benchmark.reporters),
+      // @ts-expect-error reporter is CLI flag
+      ...toArray(options.reporter),
+    ])).filter(Boolean)
+    if (reporters.length)
+      resolved.benchmark.reporters = reporters
+    else
+      resolved.benchmark.reporters = ['default']
+
+    if (options.outputFile)
+      resolved.benchmark.outputFile = options.outputFile
   }
 
   resolved.setupFiles = toArray(resolved.setupFiles || []).map(file =>
@@ -184,11 +197,13 @@ export function resolveConfig(
   if (options.related)
     resolved.related = toArray(options.related).map(file => resolve(resolved.root, file))
 
-  resolved.reporters = Array.from(new Set([
-    ...toArray(resolved.reporters),
-    // @ts-expect-error from CLI
-    ...toArray(resolved.reporter),
-  ])).filter(Boolean)
+  if (mode !== 'benchmark') {
+    resolved.reporters = Array.from(new Set([
+      ...toArray(resolved.reporters),
+      // @ts-expect-error from CLI
+      ...toArray(resolved.reporter),
+    ])).filter(Boolean)
+  }
 
   if (!resolved.reporters.length)
     resolved.reporters.push('default')

packages/vitest/src/node/reporters/benchmark/json.ts

@@ -25,7 +25,7 @@ export class JsonReporter implements Reporter {
     const tests = getTests(files)
     const numTotalTests = tests.length
     const testResults: Record<string, BenchTaskResult[]> = {}
-    const outputFile = getOutputFile(this.ctx, 'json')
+    const outputFile = getOutputFile(this.ctx.config.benchmark, 'json')
     for (const file of files) {
       const tests = getTests([file])
       for (const test of tests) {
@@ -63,7 +63,7 @@ export class JsonReporter implements Reporter {
    * @param report
    */
   async writeReport(report: string) {
-    const outputFile = getOutputFile(this.ctx, 'json')
+    const outputFile = getOutputFile(this.ctx.config.benchmark, 'json')
 
     if (outputFile) {
       const reportFile = resolve(this.ctx.config.root, outputFile)

packages/vitest/src/node/reporters/json.ts

@@ -163,7 +163,7 @@ export class JsonReporter implements Reporter {
    * @param report
    */
   async writeReport(report: string) {
-    const outputFile = getOutputFile(this.ctx, 'json')
+    const outputFile = getOutputFile(this.ctx.config, 'json')
 
     if (outputFile) {
       const reportFile = resolve(this.ctx.config.root, outputFile)

packages/vitest/src/node/reporters/junit.ts

@@ -72,7 +72,7 @@ export class JUnitReporter implements Reporter {
   async onInit(ctx: Vitest): Promise<void> {
     this.ctx = ctx
 
-    const outputFile = getOutputFile(this.ctx, 'junit')
+    const outputFile = getOutputFile(this.ctx.config, 'junit')
 
     if (outputFile) {
       this.reportFile = resolve(this.ctx.config.root, outputFile)

packages/vitest/src/types/benchmark.ts

@@ -29,6 +29,12 @@ export interface BenchmarkUserOptions {
    * and/or paths to custom reporters
    */
   reporters?: Arrayable<BenchmarkBuiltinReporters | Reporter>
+
+  /**
+   * Write test results to a file when the `--reporter=json` option is also specified.
+   * Also definable individually per reporter by using an object instead.
+   */
+  outputFile?: string | (Partial<Record<BenchmarkBuiltinReporters, string>> & Record<string, string>)
 }
 
 export interface Benchmark extends TaskBase {

packages/vitest/src/types/config.ts

@@ -508,7 +508,9 @@ export interface ResolvedConfig extends Omit<Required<UserConfig>, 'config' | 'f
 
   api?: ApiConfig
 
-  benchmark?: Required<BenchmarkUserOptions>
+  benchmark?: Required<Omit<BenchmarkUserOptions, 'outputFile'>> & {
+    outputFile?: BenchmarkUserOptions['outputFile']
+  }
 
   shard?: {
     index: number

packages/vitest/src/utils/config-helpers.ts

@@ -1,8 +1,11 @@
-import type { Vitest } from '../node/core'
 import type { BenchmarkBuiltinReporters, BuiltinReporters } from '../node/reporters'
 
-export const getOutputFile = ({ config }: Vitest, reporter: BuiltinReporters | BenchmarkBuiltinReporters) => {
-  if (!config.outputFile)
+interface PotentialConfig {
+  outputFile?: string | Partial<Record<string, string>>
+}
+
+export const getOutputFile = (config: PotentialConfig | undefined, reporter: BuiltinReporters | BenchmarkBuiltinReporters) => {
+  if (!config?.outputFile)
     return
 
   if (typeof config.outputFile === 'string')

pnpm-lock.yaml

@@ -898,7 +898,10 @@ importers:
       vitest: link:../../packages/vitest
 
   test/benchmark:
-    specifiers: {}
+    specifiers:
+      execa: ^6.1.0
+    devDependencies:
+      execa: 6.1.0
 
   test/browser:
     specifiers:

test/benchmark/package.json

@@ -3,7 +3,10 @@
   "private": true,
   "scripts": {
     "test": "node test.mjs",
-    "testu": "vitest -u",
+    "bench": "vitest bench --reporter=json",
     "coverage": "vitest run --coverage"
+  },
+  "devDependencies": {
+    "execa": "^6.1.0"
   }
 }

test/benchmark/test.mjs

@@ -1,18 +1,26 @@
 import { readFile } from 'fs/promises'
-import { startVitest } from 'vitest/node'
+import { execa } from 'execa'
 
-const success = await startVitest('benchmark', ['base.bench', 'mode.bench'], {
-  run: true,
-  update: false,
-  outputFile: './bench.json', // TODO move outputFile to benchmark
-  benchmark: {
-    reporters: ['json'],
+let error
+await execa('npx', ['vitest', 'bench', 'base.bench', 'mode.bench'], {
+  env: {
+    ...process.env,
+    CI: 'true',
+    NO_COLOR: 'true',
   },
 })
+  .catch((e) => {
+    error = e
+  })
 
 const benchResult = await readFile('./bench.json', 'utf-8')
 
 if (benchResult.includes('skip'))
   process.exit(1)
 
-process.exit(success ? 0 : 1)
+if (error) {
+  console.error(error)
+  process.exit(1)
+}
+
+process.exit(0)

test/benchmark/vitest.config.ts

@@ -2,8 +2,10 @@ import { defineConfig } from 'vitest/config'
 
 export default defineConfig({
   test: {
+    update: false,
     benchmark: {
-      reporters: 'json',
+      outputFile: './bench.json',
+      reporters: ['json'],
     },
   },
 })