diff --git a/docs/advanced/api/vitest.md b/docs/advanced/api/vitest.md index 59dd753a7..29687e371 100644 --- a/docs/advanced/api/vitest.md +++ b/docs/advanced/api/vitest.md @@ -528,3 +528,67 @@ function matchesProjectFilter(name: string): boolean Check if the name matches the current [project filter](/guide/cli#project). If there is no project filter, this will always return `true`. It is not possible to programmatically change the `--project` CLI option. + +## waitForTestRunEnd 4.0.0 {#waitfortestrunend} + +```ts +function waitForTestRunEnd(): Promise +``` + +If there is a test run happening, returns a promise that will resolve when the test run is finished. + +## createCoverageProvider 4.0.0 {#createcoverageprovider} + +```ts +function createCoverageProvider(): Promise +``` + +Creates a coverage provider if `coverage` is enabled in the config. This is done automatically if you are running tests with [`start`](#start) or [`init`](#init) methods. + +::: warning +This method will also clean all previous reports if [`coverage.clean`](/config/#coverage-clean) is not set to `false`. +::: + +## experimental_parseSpecification 4.0.0 experimental {#parsespecification} + +```ts +function experimental_parseSpecification( + specification: TestSpecification +): Promise +``` + +This function will collect all tests inside the file without running it. It uses rollup's `parseAst` function on top of Vite's `ssrTransform` to statically analyse the file and collect all tests that it can. + +::: warning +If Vitest could not analyse the name of the test, it will inject a hidden `dynamic: true` property to the test or a suite. The `id` will also have a postfix with `-dynamic` to not break tests that were collected properly. + +Vitest always injects this property in tests with `for` or `each` modifier or tests with a dynamic name (like, `hello ${property}` or `'hello' + ${property}`). Vitest will still assign a name to the test, but it cannot be used to filter the tests. + +There is nothing Vitest can do to make it possible to filter dynamic tests, but you can turn a test with `for` or `each` modifier into a name pattern with `escapeTestName` function: + +```ts +import { escapeTestName } from 'vitest/node' + +// turns into /hello, .+?/ +const escapedPattern = new RegExp(escapeTestName('hello, %s', true)) +``` +::: + +::: warning +Vitest will only collect tests defined in the file. It will never follow imports to other files. + +Vitest collects all `it`, `test`, `suite` and `describe` definitions even if they were not imported from the `vitest` entry point. +::: + +## experimental_parseSpecifications 4.0.0 experimental {#parsespecifications} + +```ts +function experimental_parseSpecifications( + specifications: TestSpecification[], + options?: { + concurrency?: number + } +): Promise +``` + +This method will [collect tests](#parsespecification) from an array of specifications. By default, Vitest will run only `os.availableParallelism()` number of specifications at a time to reduce the potential performance degradation. You can specify a different number in a second argument. diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index fa2ae30c5..b88a4bb25 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -71,6 +71,12 @@ export interface TaskBase { line: number column: number } + /** + * If the test was collected by parsing the file AST, and the name + * is not a static string, this property will be set to `true`. + * @experimental + */ + dynamic?: boolean } export interface TaskPopulated extends TaskBase { diff --git a/packages/vitest/src/node/ast-collect.ts b/packages/vitest/src/node/ast-collect.ts new file mode 100644 index 000000000..505f351cc --- /dev/null +++ b/packages/vitest/src/node/ast-collect.ts @@ -0,0 +1,477 @@ +import type { File, Suite, Task, Test } from '@vitest/runner' +import type { SourceMap } from 'node:module' +import type { TestError } from '../types/general' +import type { TestProject } from './project' +import { + calculateSuiteHash, + generateHash, + interpretTaskModes, + someTasksAreOnly, +} from '@vitest/runner/utils' +import { originalPositionFor, TraceMap } from '@vitest/utils/source-map' +import { ancestor as walkAst } from 'acorn-walk' +import { relative } from 'pathe' +import { parseAst } from 'vite' +import { createDebugger } from '../utils/debugger' + +interface ParsedFile extends File { + start: number + end: number +} + +interface ParsedTest extends Test { + start: number + end: number + dynamic: boolean +} + +interface ParsedSuite extends Suite { + start: number + end: number + dynamic: boolean +} + +interface LocalCallDefinition { + start: number + end: number + name: string + type: 'suite' | 'test' + mode: 'run' | 'skip' | 'only' | 'todo' | 'queued' + task: ParsedSuite | ParsedFile | ParsedTest + dynamic: boolean +} + +export interface FileInformation { + file: File + filepath: string + parsed: string | null + map: SourceMap | { mappings: string } | null + definitions: LocalCallDefinition[] +} + +const debug = createDebugger('vitest:ast-collect-info') +const verbose = createDebugger('vitest:ast-collect-verbose') + +function astParseFile(filepath: string, code: string) { + const ast = parseAst(code) + + if (verbose) { + verbose( + 'Collecting', + filepath, + code, + ) + } + else { + debug?.('Collecting', filepath) + } + const definitions: LocalCallDefinition[] = [] + const getName = (callee: any): string | null => { + if (!callee) { + return null + } + if (callee.type === 'Identifier') { + return callee.name + } + if (callee.type === 'CallExpression') { + return getName(callee.callee) + } + if (callee.type === 'TaggedTemplateExpression') { + return getName(callee.tag) + } + if (callee.type === 'MemberExpression') { + if ( + callee.object?.type === 'Identifier' + && ['it', 'test', 'describe', 'suite'].includes(callee.object.name) + ) { + return callee.object?.name + } + if ( + // direct call as `__vite_ssr_exports_0__.test()` + callee.object?.name?.startsWith('__vite_ssr_') + // call as `__vite_ssr_exports_0__.Vitest.test`, + // this is a special case for using Vitest namespaces popular in Effect + || (callee.object?.object?.name?.startsWith('__vite_ssr_') && callee.object?.property?.name === 'Vitest') + ) { + return getName(callee.property) + } + // call as `__vite_ssr__.test.skip()` + return getName(callee.object?.property) + } + // unwrap (0, ...) + if (callee.type === 'SequenceExpression' && callee.expressions.length === 2) { + const [e0, e1] = callee.expressions + if (e0.type === 'Literal' && e0.value === 0) { + return getName(e1) + } + } + return null + } + + walkAst(ast as any, { + CallExpression(node) { + const { callee } = node as any + const name = getName(callee) + if (!name) { + return + } + if (!['it', 'test', 'describe', 'suite'].includes(name)) { + verbose?.(`Skipping ${name} (unknown call)`) + return + } + const property = callee?.property?.name + let mode = !property || property === name ? 'run' : property + // they will be picked up in the next iteration + if (['each', 'for', 'skipIf', 'runIf'].includes(mode)) { + return + } + + let start: number + const end = node.end + // .each or (0, __vite_ssr_exports_0__.test)() + if ( + callee.type === 'CallExpression' + || callee.type === 'SequenceExpression' + || callee.type === 'TaggedTemplateExpression' + ) { + start = callee.end + } + else { + start = node.start + } + + const messageNode = node.arguments?.[0] + + if (messageNode == null) { + verbose?.(`Skipping node at ${node.start} because it doesn't have a name`) + return + } + + let message: string + if (messageNode?.type === 'Literal' || messageNode?.type === 'TemplateLiteral') { + message = code.slice(messageNode.start + 1, messageNode.end - 1) + } + else { + message = code.slice(messageNode.start, messageNode.end) + } + + if (message.startsWith('0,')) { + message = message.slice(2) + } + + message = message + // Vite SSR injects these + .replace(/__vite_ssr_import_\d+__\./g, '') + // Vitest module mocker injects these + .replace(/__vi_import_\d+__\./g, '') + + // cannot statically analyze, so we always skip it + if (mode === 'skipIf' || mode === 'runIf') { + mode = 'skip' + } + + const parentCalleeName = typeof callee?.callee === 'object' && callee?.callee.type === 'MemberExpression' && callee?.callee.property?.name + let isDynamicEach = parentCalleeName === 'each' || parentCalleeName === 'for' + if (!isDynamicEach && callee.type === 'TaggedTemplateExpression') { + const property = callee.tag?.property?.name + isDynamicEach = property === 'each' || property === 'for' + } + + debug?.('Found', name, message, `(${mode})`) + definitions.push({ + start, + end, + name: message, + type: name === 'it' || name === 'test' ? 'test' : 'suite', + mode, + task: null as any, + dynamic: isDynamicEach, + } satisfies LocalCallDefinition) + }, + }) + return { + ast, + definitions, + } +} + +export function createFailedFileTask(project: TestProject, filepath: string, error: Error): File { + const testFilepath = relative(project.config.root, filepath) + const file: ParsedFile = { + filepath, + type: 'suite', + id: /* @__PURE__ */ generateHash(`${testFilepath}${project.config.name || ''}`), + name: testFilepath, + mode: 'run', + tasks: [], + start: 0, + end: 0, + projectName: project.getName(), + meta: {}, + pool: project.browser ? 'browser' : project.config.pool, + file: null!, + result: { + state: 'fail', + errors: serializeError(project, error), + }, + } + file.file = file + return file +} + +function serializeError(ctx: TestProject, error: any): TestError[] { + if ('errors' in error && 'pluginCode' in error) { + const errors = error.errors.map((e: any) => { + return { + name: error.name, + message: e.text, + stack: e.location + ? `${error.name}: ${e.text}\n at ${relative(ctx.config.root, e.location.file)}:${e.location.line}:${e.location.column}` + : '', + } + }) + return errors + } + return [ + { + name: error.name, + stack: error.stack, + message: error.message, + }, + ] +} + +interface ParseOptions { + name: string + filepath: string + allowOnly: boolean + pool: string + testNamePattern?: RegExp | undefined +} + +function createFileTask( + testFilepath: string, + code: string, + requestMap: any, + options: ParseOptions, +) { + const { definitions, ast } = astParseFile(testFilepath, code) + const file: ParsedFile = { + filepath: options.filepath, + type: 'suite', + id: /* @__PURE__ */ generateHash(`${testFilepath}${options.name || ''}`), + name: testFilepath, + mode: 'run', + tasks: [], + start: ast.start, + end: ast.end, + projectName: options.name, + meta: {}, + pool: 'browser', + file: null!, + } + file.file = file + const indexMap = createIndexMap(code) + const map = requestMap && new TraceMap(requestMap) + let lastSuite: ParsedSuite = file as any + const updateLatestSuite = (index: number) => { + while (lastSuite.suite && lastSuite.end < index) { + lastSuite = lastSuite.suite as ParsedSuite + } + return lastSuite + } + definitions + .sort((a, b) => a.start - b.start) + .forEach((definition) => { + const latestSuite = updateLatestSuite(definition.start) + let mode = definition.mode + if (latestSuite.mode !== 'run') { + // inherit suite mode, if it's set + mode = latestSuite.mode + } + const processedLocation = indexMap.get(definition.start) + let location: { line: number; column: number } | undefined + if (map && processedLocation) { + const originalLocation = originalPositionFor(map, { + line: processedLocation.line, + column: processedLocation.column, + }) + if (originalLocation.column != null) { + verbose?.( + `Found location for`, + definition.type, + definition.name, + `${processedLocation.line}:${processedLocation.column}`, + '->', + `${originalLocation.line}:${originalLocation.column}`, + ) + location = originalLocation + } + else { + debug?.( + 'Cannot find original location for', + definition.type, + definition.name, + `${processedLocation.column}:${processedLocation.line}`, + ) + } + } + else { + debug?.( + 'Cannot find original location for', + definition.type, + definition.name, + `${definition.start}`, + ) + } + if (definition.type === 'suite') { + const task: ParsedSuite = { + type: definition.type, + id: '', + suite: latestSuite, + file, + tasks: [], + mode, + name: definition.name, + end: definition.end, + start: definition.start, + location, + dynamic: definition.dynamic, + meta: {}, + } + definition.task = task + latestSuite.tasks.push(task) + lastSuite = task + return + } + const task: ParsedTest = { + type: definition.type, + id: '', + suite: latestSuite, + file, + mode, + context: {} as any, // not used on the server + name: definition.name, + end: definition.end, + start: definition.start, + location, + dynamic: definition.dynamic, + meta: {}, + timeout: 0, + annotations: [], + } + definition.task = task + latestSuite.tasks.push(task) + }) + calculateSuiteHash(file) + const hasOnly = someTasksAreOnly(file) + interpretTaskModes( + file, + options.testNamePattern, + undefined, + hasOnly, + false, + options.allowOnly, + ) + markDynamicTests(file.tasks) + if (!file.tasks.length) { + file.result = { + state: 'fail', + errors: [ + { + name: 'Error', + message: `No test suite found in file ${options.filepath}`, + }, + ], + } + } + return file +} + +export async function astCollectTests( + project: TestProject, + filepath: string, +): Promise { + const request = await transformSSR(project, filepath) + const testFilepath = relative(project.config.root, filepath) + if (!request) { + debug?.('Cannot parse', testFilepath, '(vite didn\'t return anything)') + return createFailedFileTask( + project, + filepath, + new Error(`Failed to parse ${testFilepath}. Vite didn't return anything.`), + ) + } + return createFileTask(testFilepath, request.code, request.map, { + name: project.config.name, + filepath, + allowOnly: project.config.allowOnly, + testNamePattern: project.config.testNamePattern, + pool: project.browser ? 'browser' : project.config.pool, + }) +} + +async function transformSSR(project: TestProject, filepath: string) { + const request = await project.vite.transformRequest(filepath, { ssr: false }) + if (!request) { + return null + } + return await project.vite.ssrTransform(request.code, request.map, filepath) +} + +function createIndexMap(source: string) { + const map = new Map() + let index = 0 + let line = 1 + let column = 1 + for (const char of source) { + map.set(index++, { line, column }) + if (char === '\n' || char === '\r\n') { + line++ + column = 0 + } + else { + column++ + } + } + return map +} + +function markDynamicTests(tasks: Task[]) { + for (const task of tasks) { + if (task.dynamic) { + task.id += '-dynamic' + } + if ('tasks' in task) { + markDynamicTests(task.tasks) + } + } +} + +function escapeRegex(str: string) { + return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + +const kReplacers = new Map([ + ['%i', '\\d+?'], + ['%#', '\\d+?'], + ['%d', '[\\d.eE+-]+?'], + ['%f', '[\\d.eE+-]+?'], + ['%s', '.+?'], + ['%j', '.+?'], + ['%o', '.+?'], + ['%%', '%'], +]) + +export function escapeTestName(label: string, dynamic: boolean): string { + if (!dynamic) { + return escapeRegex(label) + } + + // Replace object access patterns ($value, $obj.a) with %s first + let pattern = label.replace(/\$[a-z_.]+/gi, '%s') + pattern = escapeRegex(pattern) + // Replace percent placeholders with their respective regex + pattern = pattern.replace(/%[i#dfsjo%]/g, m => kReplacers.get(m) || m) + return pattern +} diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index 5bc9253c2..6fddd463e 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -7,12 +7,14 @@ import type { SerializedCoverageConfig } from '../runtime/config' import type { ArgumentsType, ProvidedContext, UserConsoleLog } from '../types/general' import type { CliOptions } from './cli/cli-api' import type { ProcessPool, WorkspaceSpec } from './pool' +import type { TestModule } from './reporters/reported-tasks' import type { TestSpecification } from './spec' import type { ResolvedConfig, TestProjectConfiguration, UserConfig, VitestRunMode } from './types/config' import type { CoverageProvider } from './types/coverage' import type { Reporter } from './types/reporter' import type { TestRunResult } from './types/tests' -import { getTasks, hasFailed } from '@vitest/runner/utils' +import os from 'node:os' +import { getTasks, hasFailed, limitConcurrency } from '@vitest/runner/utils' import { SnapshotManager } from '@vitest/snapshot/manager' import { noop, toArray } from '@vitest/utils' import { normalize, relative } from 'pathe' @@ -21,6 +23,7 @@ import { WebSocketReporter } from '../api/setup' import { distDir } from '../paths' import { wildcardPatternToRegExp } from '../utils/base' import { convertTasksToEvents } from '../utils/tasks' +import { astCollectTests, createFailedFileTask } from './ast-collect' import { BrowserSessions } from './browser/sessions' import { VitestCache } from './cache' import { resolveConfig } from './config/resolveConfig' @@ -385,6 +388,20 @@ export class Vitest { return this.runner.import(moduleId) } + /** + * Creates a coverage provider if `coverage` is enabled in the config. + */ + public async createCoverageProvider(): Promise { + if (this.coverageProvider) { + return this.coverageProvider + } + const coverageProvider = await this.initCoverageProvider() + if (coverageProvider) { + await coverageProvider.clean(this.config.coverage.clean) + } + return coverageProvider || null + } + private async resolveProjects(cliOptions: UserConfig): Promise { const names = new Set() @@ -604,6 +621,17 @@ export class Vitest { return this.getModuleSpecifications(file) as WorkspaceSpec[] } + /** + * If there is a test run happening, returns a promise that will + * resolve when the test run is finished. + */ + public async waitForTestRunEnd(): Promise { + if (!this.runningPromise) { + return + } + await this.runningPromise + } + /** * Get test specifications associated with the given module. If module is not a test file, an empty array is returned. * @@ -619,6 +647,11 @@ export class Vitest { */ public clearSpecificationsCache(moduleId?: string): void { this.specifications.clearCache(moduleId) + if (!moduleId) { + this.projects.forEach((project) => { + project.testFilesList = null + }) + } } /** @@ -716,6 +749,35 @@ export class Vitest { return await this.runningPromise } + public async experimental_parseSpecifications(specifications: TestSpecification[], options?: { + /** @default os.availableParallelism() */ + concurrency?: number + }): Promise { + if (this.mode !== 'test') { + throw new Error(`The \`experimental_parseSpecifications\` does not support "${this.mode}" mode.`) + } + const concurrency = options?.concurrency ?? (typeof os.availableParallelism === 'function' + ? os.availableParallelism() + : os.cpus().length) + const limit = limitConcurrency(concurrency) + const promises = specifications.map(specification => + limit(() => this.experimental_parseSpecification(specification)), + ) + return Promise.all(promises) + } + + public async experimental_parseSpecification(specification: TestSpecification): Promise { + if (this.mode !== 'test') { + throw new Error(`The \`experimental_parseSpecification\` does not support "${this.mode}" mode.`) + } + const file = await astCollectTests(specification.project, specification.moduleId).catch((error) => { + return createFailedFileTask(specification.project, specification.moduleId, error) + }) + // register in state, so it can be retrieved by "getReportedEntity" + this.state.collectFiles(specification.project, [file]) + return this.state.getReportedEntity(file) as TestModule + } + /** * Collect tests in specified modules. Vitest will run the files to collect tests. * @param specifications A list of specifications to run. diff --git a/packages/vitest/src/node/project.ts b/packages/vitest/src/node/project.ts index 56f05e867..121b3127b 100644 --- a/packages/vitest/src/node/project.ts +++ b/packages/vitest/src/node/project.ts @@ -68,12 +68,12 @@ export class TestProject { /** @internal */ _vite?: ViteDevServer /** @internal */ _hash?: string /** @internal */ _resolver!: VitestResolver + /** @inetrnal */ testFilesList: string[] | null = null private runner!: ModuleRunner private closingPromise: Promise | undefined - private testFilesList: string[] | null = null private typecheckFilesList: string[] | null = null private _globalSetups?: GlobalSetupFile[] diff --git a/packages/vitest/src/node/reporters/reported-tasks.ts b/packages/vitest/src/node/reporters/reported-tasks.ts index e76735e65..a8cc24172 100644 --- a/packages/vitest/src/node/reporters/reported-tasks.ts +++ b/packages/vitest/src/node/reporters/reported-tasks.ts @@ -681,3 +681,11 @@ function getSuiteState(task: RunnerTestSuite | RunnerTestFile): TestSuiteState { } throw new Error(`Unknown suite state: ${state}`) } + +export function experimental_getRunnerTask(entity: TestCase): RunnerTestCase +export function experimental_getRunnerTask(entity: TestSuite): RunnerTestSuite +export function experimental_getRunnerTask(entity: TestModule): RunnerTestFile +export function experimental_getRunnerTask(entity: TestCase | TestSuite | TestModule): RunnerTestSuite | RunnerTestFile | RunnerTestCase +export function experimental_getRunnerTask(entity: TestCase | TestSuite | TestModule): RunnerTestSuite | RunnerTestFile | RunnerTestCase { + return entity.task +} diff --git a/packages/vitest/src/public/node.ts b/packages/vitest/src/public/node.ts index 6e95b826a..43e99abb5 100644 --- a/packages/vitest/src/public/node.ts +++ b/packages/vitest/src/public/node.ts @@ -4,6 +4,7 @@ import { Vitest } from '../node/core' export const version: string = Vitest.version export { isValidApiRequest } from '../api/check' +export { escapeTestName } from '../node/ast-collect' export { parseCLI } from '../node/cli/cac' export type { CliParseOptions } from '../node/cli/cac' export { startVitest } from '../node/cli/cli-api' @@ -47,6 +48,7 @@ export type { TestSuite, TestSuiteState, } from '../node/reporters/reported-tasks' +export { experimental_getRunnerTask } from '../node/reporters/reported-tasks' export { BaseSequencer } from '../node/sequencers/BaseSequencer' export type { diff --git a/test/core/test/exports.test.ts b/test/core/test/exports.test.ts index ae146d001..93de0dc4a 100644 --- a/test/core/test/exports.test.ts +++ b/test/core/test/exports.test.ts @@ -102,6 +102,8 @@ it('exports snapshot', async ({ skip, task }) => { "createVitest": "function", "distDir": "string", "esbuildVersion": "string", + "escapeTestName": "function", + "experimental_getRunnerTask": "function", "generateFileHash": "function", "getFilePoolName": "function", "isCSSRequest": "function", @@ -259,6 +261,8 @@ it('exports snapshot', async ({ skip, task }) => { "createVitest": "function", "distDir": "string", "esbuildVersion": "string", + "escapeTestName": "function", + "experimental_getRunnerTask": "function", "generateFileHash": "function", "getFilePoolName": "function", "isCSSRequest": "function",