feat(api): expose experimental_parseSpecifications (#8408)

docs/advanced/api/vitest.md

@@ -528,3 +528,67 @@ function matchesProjectFilter(name: string): boolean
 Check if the name matches the current [project filter](/guide/cli#project). If there is no project filter, this will always return `true`.
 
 It is not possible to programmatically change the `--project` CLI option.
+
+## waitForTestRunEnd <Version>4.0.0</Version> {#waitfortestrunend}
+
+```ts
+function waitForTestRunEnd(): Promise<void>
+```
+
+If there is a test run happening, returns a promise that will resolve when the test run is finished.
+
+## createCoverageProvider <Version>4.0.0</Version> {#createcoverageprovider}
+
+```ts
+function createCoverageProvider(): Promise<CoverageProvider | null>
+```
+
+Creates a coverage provider if `coverage` is enabled in the config. This is done automatically if you are running tests with [`start`](#start) or [`init`](#init) methods.
+
+::: warning
+This method will also clean all previous reports if [`coverage.clean`](/config/#coverage-clean) is not set to `false`.
+:::
+
+## experimental_parseSpecification <Version>4.0.0</Version> <Badge type="warning">experimental</Badge> {#parsespecification}
+
+```ts
+function experimental_parseSpecification(
+  specification: TestSpecification
+): Promise<TestModule>
+```
+
+This function will collect all tests inside the file without running it. It uses rollup's `parseAst` function on top of Vite's `ssrTransform` to statically analyse the file and collect all tests that it can.
+
+::: warning
+If Vitest could not analyse the name of the test, it will inject a hidden `dynamic: true` property to the test or a suite. The `id` will also have a postfix with `-dynamic` to not break tests that were collected properly.
+
+Vitest always injects this property in tests with `for` or `each` modifier or tests with a dynamic name (like, `hello ${property}` or `'hello' + ${property}`). Vitest will still assign a name to the test, but it cannot be used to filter the tests.
+
+There is nothing Vitest can do to make it possible to filter dynamic tests, but you can turn a test with `for` or `each` modifier into a name pattern with `escapeTestName` function:
+
+```ts
+import { escapeTestName } from 'vitest/node'
+
+// turns into /hello, .+?/
+const escapedPattern = new RegExp(escapeTestName('hello, %s', true))
+```
+:::
+
+::: warning
+Vitest will only collect tests defined in the file. It will never follow imports to other files.
+
+Vitest collects all `it`, `test`, `suite` and `describe` definitions even if they were not imported from the `vitest` entry point.
+:::
+
+## experimental_parseSpecifications <Version>4.0.0</Version> <Badge type="warning">experimental</Badge> {#parsespecifications}
+
+```ts
+function experimental_parseSpecifications(
+  specifications: TestSpecification[],
+  options?: {
+    concurrency?: number
+  }
+): Promise<TestModule[]>
+```
+
+This method will [collect tests](#parsespecification) from an array of specifications. By default, Vitest will run only `os.availableParallelism()` number of specifications at a time to reduce the potential performance degradation. You can specify a different number in a second argument.

packages/runner/src/types/tasks.ts

@@ -71,6 +71,12 @@ export interface TaskBase {
     line: number
     column: number
   }
+  /**
+   * If the test was collected by parsing the file AST, and the name
+   * is not a static string, this property will be set to `true`.
+   * @experimental
+   */
+  dynamic?: boolean
 }
 
 export interface TaskPopulated extends TaskBase {

packages/vitest/src/node/ast-collect.ts

@@ -0,0 +1,477 @@
+import type { File, Suite, Task, Test } from '@vitest/runner'
+import type { SourceMap } from 'node:module'
+import type { TestError } from '../types/general'
+import type { TestProject } from './project'
+import {
+  calculateSuiteHash,
+  generateHash,
+  interpretTaskModes,
+  someTasksAreOnly,
+} from '@vitest/runner/utils'
+import { originalPositionFor, TraceMap } from '@vitest/utils/source-map'
+import { ancestor as walkAst } from 'acorn-walk'
+import { relative } from 'pathe'
+import { parseAst } from 'vite'
+import { createDebugger } from '../utils/debugger'
+
+interface ParsedFile extends File {
+  start: number
+  end: number
+}
+
+interface ParsedTest extends Test {
+  start: number
+  end: number
+  dynamic: boolean
+}
+
+interface ParsedSuite extends Suite {
+  start: number
+  end: number
+  dynamic: boolean
+}
+
+interface LocalCallDefinition {
+  start: number
+  end: number
+  name: string
+  type: 'suite' | 'test'
+  mode: 'run' | 'skip' | 'only' | 'todo' | 'queued'
+  task: ParsedSuite | ParsedFile | ParsedTest
+  dynamic: boolean
+}
+
+export interface FileInformation {
+  file: File
+  filepath: string
+  parsed: string | null
+  map: SourceMap | { mappings: string } | null
+  definitions: LocalCallDefinition[]
+}
+
+const debug = createDebugger('vitest:ast-collect-info')
+const verbose = createDebugger('vitest:ast-collect-verbose')
+
+function astParseFile(filepath: string, code: string) {
+  const ast = parseAst(code)
+
+  if (verbose) {
+    verbose(
+      'Collecting',
+      filepath,
+      code,
+    )
+  }
+  else {
+    debug?.('Collecting', filepath)
+  }
+  const definitions: LocalCallDefinition[] = []
+  const getName = (callee: any): string | null => {
+    if (!callee) {
+      return null
+    }
+    if (callee.type === 'Identifier') {
+      return callee.name
+    }
+    if (callee.type === 'CallExpression') {
+      return getName(callee.callee)
+    }
+    if (callee.type === 'TaggedTemplateExpression') {
+      return getName(callee.tag)
+    }
+    if (callee.type === 'MemberExpression') {
+      if (
+        callee.object?.type === 'Identifier'
+        && ['it', 'test', 'describe', 'suite'].includes(callee.object.name)
+      ) {
+        return callee.object?.name
+      }
+      if (
+        // direct call as `__vite_ssr_exports_0__.test()`
+        callee.object?.name?.startsWith('__vite_ssr_')
+        // call as `__vite_ssr_exports_0__.Vitest.test`,
+        // this is a special case for using Vitest namespaces popular in Effect
+        || (callee.object?.object?.name?.startsWith('__vite_ssr_') && callee.object?.property?.name === 'Vitest')
+      ) {
+        return getName(callee.property)
+      }
+      // call as `__vite_ssr__.test.skip()`
+      return getName(callee.object?.property)
+    }
+    // unwrap (0, ...)
+    if (callee.type === 'SequenceExpression' && callee.expressions.length === 2) {
+      const [e0, e1] = callee.expressions
+      if (e0.type === 'Literal' && e0.value === 0) {
+        return getName(e1)
+      }
+    }
+    return null
+  }
+
+  walkAst(ast as any, {
+    CallExpression(node) {
+      const { callee } = node as any
+      const name = getName(callee)
+      if (!name) {
+        return
+      }
+      if (!['it', 'test', 'describe', 'suite'].includes(name)) {
+        verbose?.(`Skipping ${name} (unknown call)`)
+        return
+      }
+      const property = callee?.property?.name
+      let mode = !property || property === name ? 'run' : property
+      // they will be picked up in the next iteration
+      if (['each', 'for', 'skipIf', 'runIf'].includes(mode)) {
+        return
+      }
+
+      let start: number
+      const end = node.end
+      // .each or (0, __vite_ssr_exports_0__.test)()
+      if (
+        callee.type === 'CallExpression'
+        || callee.type === 'SequenceExpression'
+        || callee.type === 'TaggedTemplateExpression'
+      ) {
+        start = callee.end
+      }
+      else {
+        start = node.start
+      }
+
+      const messageNode = node.arguments?.[0]
+
+      if (messageNode == null) {
+        verbose?.(`Skipping node at ${node.start} because it doesn't have a name`)
+        return
+      }
+
+      let message: string
+      if (messageNode?.type === 'Literal' || messageNode?.type === 'TemplateLiteral') {
+        message = code.slice(messageNode.start + 1, messageNode.end - 1)
+      }
+      else {
+        message = code.slice(messageNode.start, messageNode.end)
+      }
+
+      if (message.startsWith('0,')) {
+        message = message.slice(2)
+      }
+
+      message = message
+        // Vite SSR injects these
+        .replace(/__vite_ssr_import_\d+__\./g, '')
+        // Vitest module mocker injects these
+        .replace(/__vi_import_\d+__\./g, '')
+
+      // cannot statically analyze, so we always skip it
+      if (mode === 'skipIf' || mode === 'runIf') {
+        mode = 'skip'
+      }
+
+      const parentCalleeName = typeof callee?.callee === 'object' && callee?.callee.type === 'MemberExpression' && callee?.callee.property?.name
+      let isDynamicEach = parentCalleeName === 'each' || parentCalleeName === 'for'
+      if (!isDynamicEach && callee.type === 'TaggedTemplateExpression') {
+        const property = callee.tag?.property?.name
+        isDynamicEach = property === 'each' || property === 'for'
+      }
+
+      debug?.('Found', name, message, `(${mode})`)
+      definitions.push({
+        start,
+        end,
+        name: message,
+        type: name === 'it' || name === 'test' ? 'test' : 'suite',
+        mode,
+        task: null as any,
+        dynamic: isDynamicEach,
+      } satisfies LocalCallDefinition)
+    },
+  })
+  return {
+    ast,
+    definitions,
+  }
+}
+
+export function createFailedFileTask(project: TestProject, filepath: string, error: Error): File {
+  const testFilepath = relative(project.config.root, filepath)
+  const file: ParsedFile = {
+    filepath,
+    type: 'suite',
+    id: /* @__PURE__ */ generateHash(`${testFilepath}${project.config.name || ''}`),
+    name: testFilepath,
+    mode: 'run',
+    tasks: [],
+    start: 0,
+    end: 0,
+    projectName: project.getName(),
+    meta: {},
+    pool: project.browser ? 'browser' : project.config.pool,
+    file: null!,
+    result: {
+      state: 'fail',
+      errors: serializeError(project, error),
+    },
+  }
+  file.file = file
+  return file
+}
+
+function serializeError(ctx: TestProject, error: any): TestError[] {
+  if ('errors' in error && 'pluginCode' in error) {
+    const errors = error.errors.map((e: any) => {
+      return {
+        name: error.name,
+        message: e.text,
+        stack: e.location
+          ? `${error.name}: ${e.text}\n  at ${relative(ctx.config.root, e.location.file)}:${e.location.line}:${e.location.column}`
+          : '',
+      }
+    })
+    return errors
+  }
+  return [
+    {
+      name: error.name,
+      stack: error.stack,
+      message: error.message,
+    },
+  ]
+}
+
+interface ParseOptions {
+  name: string
+  filepath: string
+  allowOnly: boolean
+  pool: string
+  testNamePattern?: RegExp | undefined
+}
+
+function createFileTask(
+  testFilepath: string,
+  code: string,
+  requestMap: any,
+  options: ParseOptions,
+) {
+  const { definitions, ast } = astParseFile(testFilepath, code)
+  const file: ParsedFile = {
+    filepath: options.filepath,
+    type: 'suite',
+    id: /* @__PURE__ */ generateHash(`${testFilepath}${options.name || ''}`),
+    name: testFilepath,
+    mode: 'run',
+    tasks: [],
+    start: ast.start,
+    end: ast.end,
+    projectName: options.name,
+    meta: {},
+    pool: 'browser',
+    file: null!,
+  }
+  file.file = file
+  const indexMap = createIndexMap(code)
+  const map = requestMap && new TraceMap(requestMap)
+  let lastSuite: ParsedSuite = file as any
+  const updateLatestSuite = (index: number) => {
+    while (lastSuite.suite && lastSuite.end < index) {
+      lastSuite = lastSuite.suite as ParsedSuite
+    }
+    return lastSuite
+  }
+  definitions
+    .sort((a, b) => a.start - b.start)
+    .forEach((definition) => {
+      const latestSuite = updateLatestSuite(definition.start)
+      let mode = definition.mode
+      if (latestSuite.mode !== 'run') {
+        // inherit suite mode, if it's set
+        mode = latestSuite.mode
+      }
+      const processedLocation = indexMap.get(definition.start)
+      let location: { line: number; column: number } | undefined
+      if (map && processedLocation) {
+        const originalLocation = originalPositionFor(map, {
+          line: processedLocation.line,
+          column: processedLocation.column,
+        })
+        if (originalLocation.column != null) {
+          verbose?.(
+            `Found location for`,
+            definition.type,
+            definition.name,
+            `${processedLocation.line}:${processedLocation.column}`,
+            '->',
+            `${originalLocation.line}:${originalLocation.column}`,
+          )
+          location = originalLocation
+        }
+        else {
+          debug?.(
+            'Cannot find original location for',
+            definition.type,
+            definition.name,
+            `${processedLocation.column}:${processedLocation.line}`,
+          )
+        }
+      }
+      else {
+        debug?.(
+          'Cannot find original location for',
+          definition.type,
+          definition.name,
+          `${definition.start}`,
+        )
+      }
+      if (definition.type === 'suite') {
+        const task: ParsedSuite = {
+          type: definition.type,
+          id: '',
+          suite: latestSuite,
+          file,
+          tasks: [],
+          mode,
+          name: definition.name,
+          end: definition.end,
+          start: definition.start,
+          location,
+          dynamic: definition.dynamic,
+          meta: {},
+        }
+        definition.task = task
+        latestSuite.tasks.push(task)
+        lastSuite = task
+        return
+      }
+      const task: ParsedTest = {
+        type: definition.type,
+        id: '',
+        suite: latestSuite,
+        file,
+        mode,
+        context: {} as any, // not used on the server
+        name: definition.name,
+        end: definition.end,
+        start: definition.start,
+        location,
+        dynamic: definition.dynamic,
+        meta: {},
+        timeout: 0,
+        annotations: [],
+      }
+      definition.task = task
+      latestSuite.tasks.push(task)
+    })
+  calculateSuiteHash(file)
+  const hasOnly = someTasksAreOnly(file)
+  interpretTaskModes(
+    file,
+    options.testNamePattern,
+    undefined,
+    hasOnly,
+    false,
+    options.allowOnly,
+  )
+  markDynamicTests(file.tasks)
+  if (!file.tasks.length) {
+    file.result = {
+      state: 'fail',
+      errors: [
+        {
+          name: 'Error',
+          message: `No test suite found in file ${options.filepath}`,
+        },
+      ],
+    }
+  }
+  return file
+}
+
+export async function astCollectTests(
+  project: TestProject,
+  filepath: string,
+): Promise<File> {
+  const request = await transformSSR(project, filepath)
+  const testFilepath = relative(project.config.root, filepath)
+  if (!request) {
+    debug?.('Cannot parse', testFilepath, '(vite didn\'t return anything)')
+    return createFailedFileTask(
+      project,
+      filepath,
+      new Error(`Failed to parse ${testFilepath}. Vite didn't return anything.`),
+    )
+  }
+  return createFileTask(testFilepath, request.code, request.map, {
+    name: project.config.name,
+    filepath,
+    allowOnly: project.config.allowOnly,
+    testNamePattern: project.config.testNamePattern,
+    pool: project.browser ? 'browser' : project.config.pool,
+  })
+}
+
+async function transformSSR(project: TestProject, filepath: string) {
+  const request = await project.vite.transformRequest(filepath, { ssr: false })
+  if (!request) {
+    return null
+  }
+  return await project.vite.ssrTransform(request.code, request.map, filepath)
+}
+
+function createIndexMap(source: string) {
+  const map = new Map<number, { line: number; column: number }>()
+  let index = 0
+  let line = 1
+  let column = 1
+  for (const char of source) {
+    map.set(index++, { line, column })
+    if (char === '\n' || char === '\r\n') {
+      line++
+      column = 0
+    }
+    else {
+      column++
+    }
+  }
+  return map
+}
+
+function markDynamicTests(tasks: Task[]) {
+  for (const task of tasks) {
+    if (task.dynamic) {
+      task.id += '-dynamic'
+    }
+    if ('tasks' in task) {
+      markDynamicTests(task.tasks)
+    }
+  }
+}
+
+function escapeRegex(str: string) {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+const kReplacers = new Map<string, string>([
+  ['%i', '\\d+?'],
+  ['%#', '\\d+?'],
+  ['%d', '[\\d.eE+-]+?'],
+  ['%f', '[\\d.eE+-]+?'],
+  ['%s', '.+?'],
+  ['%j', '.+?'],
+  ['%o', '.+?'],
+  ['%%', '%'],
+])
+
+export function escapeTestName(label: string, dynamic: boolean): string {
+  if (!dynamic) {
+    return escapeRegex(label)
+  }
+
+  // Replace object access patterns ($value, $obj.a) with %s first
+  let pattern = label.replace(/\$[a-z_.]+/gi, '%s')
+  pattern = escapeRegex(pattern)
+  // Replace percent placeholders with their respective regex
+  pattern = pattern.replace(/%[i#dfsjo%]/g, m => kReplacers.get(m) || m)
+  return pattern
+}

packages/vitest/src/node/core.ts

@@ -7,12 +7,14 @@ import type { SerializedCoverageConfig } from '../runtime/config'
 import type { ArgumentsType, ProvidedContext, UserConsoleLog } from '../types/general'
 import type { CliOptions } from './cli/cli-api'
 import type { ProcessPool, WorkspaceSpec } from './pool'
+import type { TestModule } from './reporters/reported-tasks'
 import type { TestSpecification } from './spec'
 import type { ResolvedConfig, TestProjectConfiguration, UserConfig, VitestRunMode } from './types/config'
 import type { CoverageProvider } from './types/coverage'
 import type { Reporter } from './types/reporter'
 import type { TestRunResult } from './types/tests'
-import { getTasks, hasFailed } from '@vitest/runner/utils'
+import os from 'node:os'
+import { getTasks, hasFailed, limitConcurrency } from '@vitest/runner/utils'
 import { SnapshotManager } from '@vitest/snapshot/manager'
 import { noop, toArray } from '@vitest/utils'
 import { normalize, relative } from 'pathe'
@@ -21,6 +23,7 @@ import { WebSocketReporter } from '../api/setup'
 import { distDir } from '../paths'
 import { wildcardPatternToRegExp } from '../utils/base'
 import { convertTasksToEvents } from '../utils/tasks'
+import { astCollectTests, createFailedFileTask } from './ast-collect'
 import { BrowserSessions } from './browser/sessions'
 import { VitestCache } from './cache'
 import { resolveConfig } from './config/resolveConfig'
@@ -385,6 +388,20 @@ export class Vitest {
     return this.runner.import(moduleId)
   }
 
+  /**
+   * Creates a coverage provider if `coverage` is enabled in the config.
+   */
+  public async createCoverageProvider(): Promise<CoverageProvider | null> {
+    if (this.coverageProvider) {
+      return this.coverageProvider
+    }
+    const coverageProvider = await this.initCoverageProvider()
+    if (coverageProvider) {
+      await coverageProvider.clean(this.config.coverage.clean)
+    }
+    return coverageProvider || null
+  }
+
   private async resolveProjects(cliOptions: UserConfig): Promise<TestProject[]> {
     const names = new Set<string>()
 
@@ -604,6 +621,17 @@ export class Vitest {
     return this.getModuleSpecifications(file) as WorkspaceSpec[]
   }
 
+  /**
+   * If there is a test run happening, returns a promise that will
+   * resolve when the test run is finished.
+   */
+  public async waitForTestRunEnd(): Promise<void> {
+    if (!this.runningPromise) {
+      return
+    }
+    await this.runningPromise
+  }
+
   /**
    * Get test specifications associated with the given module. If module is not a test file, an empty array is returned.
    *
@@ -619,6 +647,11 @@ export class Vitest {
    */
   public clearSpecificationsCache(moduleId?: string): void {
     this.specifications.clearCache(moduleId)
+    if (!moduleId) {
+      this.projects.forEach((project) => {
+        project.testFilesList = null
+      })
+    }
   }
 
   /**
@@ -716,6 +749,35 @@ export class Vitest {
     return await this.runningPromise
   }
 
+  public async experimental_parseSpecifications(specifications: TestSpecification[], options?: {
+    /** @default os.availableParallelism() */
+    concurrency?: number
+  }): Promise<TestModule[]> {
+    if (this.mode !== 'test') {
+      throw new Error(`The \`experimental_parseSpecifications\` does not support "${this.mode}" mode.`)
+    }
+    const concurrency = options?.concurrency ?? (typeof os.availableParallelism === 'function'
+      ? os.availableParallelism()
+      : os.cpus().length)
+    const limit = limitConcurrency(concurrency)
+    const promises = specifications.map(specification =>
+      limit(() => this.experimental_parseSpecification(specification)),
+    )
+    return Promise.all(promises)
+  }
+
+  public async experimental_parseSpecification(specification: TestSpecification): Promise<TestModule> {
+    if (this.mode !== 'test') {
+      throw new Error(`The \`experimental_parseSpecification\` does not support "${this.mode}" mode.`)
+    }
+    const file = await astCollectTests(specification.project, specification.moduleId).catch((error) => {
+      return createFailedFileTask(specification.project, specification.moduleId, error)
+    })
+    // register in state, so it can be retrieved by "getReportedEntity"
+    this.state.collectFiles(specification.project, [file])
+    return this.state.getReportedEntity(file) as TestModule
+  }
+
   /**
    * Collect tests in specified modules. Vitest will run the files to collect tests.
    * @param specifications A list of specifications to run.

packages/vitest/src/node/project.ts

@@ -68,12 +68,12 @@ export class TestProject {
   /** @internal */ _vite?: ViteDevServer
   /** @internal */ _hash?: string
   /** @internal */ _resolver!: VitestResolver
+  /** @inetrnal */ testFilesList: string[] | null = null
 
   private runner!: ModuleRunner
 
   private closingPromise: Promise<void> | undefined
 
-  private testFilesList: string[] | null = null
   private typecheckFilesList: string[] | null = null
 
   private _globalSetups?: GlobalSetupFile[]

packages/vitest/src/node/reporters/reported-tasks.ts

@@ -681,3 +681,11 @@ function getSuiteState(task: RunnerTestSuite | RunnerTestFile): TestSuiteState {
   }
   throw new Error(`Unknown suite state: ${state}`)
 }
+
+export function experimental_getRunnerTask(entity: TestCase): RunnerTestCase
+export function experimental_getRunnerTask(entity: TestSuite): RunnerTestSuite
+export function experimental_getRunnerTask(entity: TestModule): RunnerTestFile
+export function experimental_getRunnerTask(entity: TestCase | TestSuite | TestModule): RunnerTestSuite | RunnerTestFile | RunnerTestCase
+export function experimental_getRunnerTask(entity: TestCase | TestSuite | TestModule): RunnerTestSuite | RunnerTestFile | RunnerTestCase {
+  return entity.task
+}

packages/vitest/src/public/node.ts

@@ -4,6 +4,7 @@ import { Vitest } from '../node/core'
 export const version: string = Vitest.version
 
 export { isValidApiRequest } from '../api/check'
+export { escapeTestName } from '../node/ast-collect'
 export { parseCLI } from '../node/cli/cac'
 export type { CliParseOptions } from '../node/cli/cac'
 export { startVitest } from '../node/cli/cli-api'
@@ -47,6 +48,7 @@ export type {
   TestSuite,
   TestSuiteState,
 } from '../node/reporters/reported-tasks'
+export { experimental_getRunnerTask } from '../node/reporters/reported-tasks'
 export { BaseSequencer } from '../node/sequencers/BaseSequencer'
 
 export type {

test/core/test/exports.test.ts

@@ -102,6 +102,8 @@ it('exports snapshot', async ({ skip, task }) => {
           "createVitest": "function",
           "distDir": "string",
           "esbuildVersion": "string",
+          "escapeTestName": "function",
+          "experimental_getRunnerTask": "function",
           "generateFileHash": "function",
           "getFilePoolName": "function",
           "isCSSRequest": "function",
@@ -259,6 +261,8 @@ it('exports snapshot', async ({ skip, task }) => {
             "createVitest": "function",
             "distDir": "string",
             "esbuildVersion": "string",
+            "escapeTestName": "function",
+            "experimental_getRunnerTask": "function",
             "generateFileHash": "function",
             "getFilePoolName": "function",
             "isCSSRequest": "function",