perf(experimental): add file system cache (#9026)

This commit is contained in:
Vladimir 2025-11-20 11:29:19 +01:00 committed by GitHub
parent 8508296e9a
commit 1b14737124
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
43 changed files with 1624 additions and 310 deletions

View File

@ -127,6 +127,42 @@ jobs:
path: test/ui/test-results/
retention-days: 30
test-cached:
needs: changed
name: 'Cache&Test: node-${{ matrix.node_version }}, ${{ matrix.os }}'
if: needs.changed.outputs.should_skip != 'true'
runs-on: ${{ matrix.os }}
timeout-minutes: 30
strategy:
matrix:
node_version: [24]
os:
- macos-latest
- windows-latest
fail-fast: false
steps:
- uses: actions/checkout@v5
- uses: ./.github/actions/setup-and-cache
with:
node-version: ${{ matrix.node_version }}
- uses: browser-actions/setup-chrome@b94431e051d1c52dcbe9a7092a4f10f827795416 # v2.1.0
- name: Install
run: pnpm i
- uses: ./.github/actions/setup-playwright
- name: Build
run: pnpm run build
- name: Test
run: pnpm run test:ci:cache
test-browser:
needs: changed
name: 'Browsers: node-${{ matrix.node_version }}, ${{ matrix.os }}'

View File

@ -41,6 +41,7 @@ const skipConfig = new Set([
'ui',
'browser.name',
'browser.fileParallelism',
'clearCache',
])
function resolveOptions(options: CLIOptions<any>, parentName?: string) {

View File

@ -123,3 +123,51 @@ The project's `configFile` can be accessed in Vite's config: `project.vite.confi
Note that this will also inherit the `name` - Vitest doesn't allow multiple projects with the same name, so this will throw an error. Make sure you specified a different name. You can access the current name via the `project.name` property and all used names are available in the `vitest.projects` array.
:::
### experimental_defineCacheKeyGenerator <Version type="experimental">4.0.11</Version> <Experimental /> {#definecachekeygenerator}
```ts
interface CacheKeyIdGeneratorContext {
environment: DevEnvironment
id: string
sourceCode: string
}
function experimental_defineCacheKeyGenerator(
callback: (context: CacheKeyIdGeneratorContext) => string | undefined | null | false
): void
```
Define a generator that will be applied before hashing the cache key.
Use this to make sure Vitest generates correct hash. It is a good idea to define this function if your plugin can be registered with different options.
This is called only if [`experimental.fsModuleCache`](/config/experimental#fsmodulecache) is defined.
```ts
interface PluginOptions {
replacePropertyKey: string
replacePropertyValue: string
}
export function plugin(options: PluginOptions) {
return {
name: 'plugin-that-replaces-property',
transform(code) {
return code.replace(
options.replacePropertyKey,
options.replacePropertyValue
)
},
configureVitest({ experimental_defineCacheKeyGenerator }) {
experimental_defineCacheKeyGenerator(() => {
// since these options affect the transform result,
// return them together as a unique string
return options.replacePropertyKey + options.replacePropertyValue
})
}
}
}
```
If the `false` is returned, the module will not be cached on the file system.

View File

@ -607,3 +607,11 @@ function experimental_parseSpecifications(
```
This method will [collect tests](#parsespecification) from an array of specifications. By default, Vitest will run only `os.availableParallelism()` number of specifications at a time to reduce the potential performance degradation. You can specify a different number in a second argument.
## experimental_clearCache <Version type="experimental">4.0.11</Version> <Badge type="warning">experimental</Badge> {#clearcache}
```ts
function experimental_clearCache(): Promise<void>
```
Deletes all Vitest caches, including [`experimental.fsModuleCache`](/config/experimental#fsmodulecache).

View File

@ -5,7 +5,75 @@ outline: deep
# experimental
## openTelemetry <Version type="experimental">4.0.10</Version>
## experimental.fsModuleCache <Version type="experimental">4.0.11</Version> {#experimental-fsmodulecache}
- **Type:** `boolean`
- **Default:** `false`
Enabling this option allows Vitest to keep cached modules on the file system, making tests run faster between reruns.
You can delete the old cache by running [`vitest --clearCache`](/guide/cli#clearcache).
::: warning BROWSER SUPPORT
At the moment, this option does not affect [the browser](/guide/browser/).
:::
You can debug if your modules are cached by running vitest with a `DEBUG=vitest:cache:fs` environment variable:
```shell
DEBUG=vitest:cache:fs vitest --experimental.fsModuleCache
```
### Known Issues
Vitest creates persistent file hash based on file content, its id, vite's environment configuration and coverage status. Vitest tries to use as much information it has about the configuration, but it is still incomplete. At the moment, it is not possible to track your plugin options because there is no standard interface for it.
If you have a plugin that relies on things outside the file content or the public configuration (like reading another file or a folder), it's possible that the cache will get stale. To workaround that, you can define a [cache key generator](/api/advanced/plugin#definecachekeygenerator) to specify dynamic option or to opt-out of caching for that module:
```js [vitest.config.js]
import { defineConfig } from 'vitest/config'
export default defineConfig({
plugins: [
{
name: 'vitest-cache',
configureVitest({ experimental_defineCacheKeyGenerator }) {
experimental_defineCacheKeyGenerator(({ id, sourceCode }) => {
// never cache this id
if (id.includes('do-not-cache')) {
return false
}
// cache this file based on the value of a dynamic variable
if (sourceCode.includes('myDynamicVar')) {
return process.env.DYNAMIC_VAR_VALUE
}
})
}
}
],
test: {
experimental: {
fsModuleCache: true,
},
},
})
```
If you are a plugin author, consider defining a [cache key generator](/api/advanced/plugin#definecachekeygenerator) in your plugin if it can be registered with different options that affect the transform result.
## experimental.fsModuleCachePath <Version type="experimental">4.0.11</Version> {#experimental-fsmodulecachepath}
- **Type:** `string`
- **Default:** `'node_modules/.experimental-vitest-cache'`
Directory where the file system cache is located.
By default, Vitest will try to find the workspace root and store the cache inside the `node_modules` folder. The root is based on your package manager's lockfile (for example, `.package-lock.json`, `.yarn-state.yml`, `.pnpm/lock.yaml` and so on).
At the moment, Vitest ignores the [test.cache.dir](/config/cache) or [cacheDir](https://vite.dev/config/shared-options#cachedir) options completely and creates a separate folder.
## experimental.openTelemetry <Version type="experimental">4.0.11</Version> {#experimental-opentelemetry}
- **Type:**

View File

@ -71,25 +71,3 @@ If a `RegExp` is provided, it is matched against the full file path.
When a dependency is a valid ESM package, try to guess the cjs version based on the path. This might be helpful, if a dependency has the wrong ESM file.
This might potentially cause some misalignment if a package has different logic in ESM and CJS mode.
## debug
### dump
- **Type:** `string | boolean`
- **Default:** `false`
The folder where Vitest stores the contents of inlined test files that can be inspected manually.
If set to `true`, Vitest dumps the files inside the `.vitest-dump` folder relative to the root of the project.
You can also use `VITEST_DEBUG_DUMP` env variable to enable this conditionally.
### load
- **Type:** `boolean`
- **Default:** `false`
Read files from the dump instead of transforming them. If dump is disabled, this does nothing.
You can also use `VITEST_DEBUG_LOAD_DUMP` env variable to enable this conditionally.

View File

@ -798,3 +798,16 @@ Use `bundle` to bundle the config with esbuild or `runner` (experimental) to pro
- **CLI:** `--standalone`
Start Vitest without running tests. Tests will be running only on change. This option is ignored when CLI file filters are passed. (default: `false`)
### clearCache
- **CLI:** `--clearCache`
Delete all Vitest caches, including `experimental.fsModuleCache`, without running any tests. This will reduce the performance in the subsequent test run.
### experimental.fsModuleCache
- **CLI:** `--experimental.fsModuleCache`
- **Config:** [experimental.fsModuleCache](/config/experimental#experimental-fsmodulecache)
Enable caching of modules on the file system between reruns.

View File

@ -25,6 +25,7 @@
"release": "tsx scripts/release.ts",
"test": "pnpm --filter test-core test:threads",
"test:ci": "CI=true pnpm -r --reporter-hide-prefix --stream --sequential --filter '@vitest/test-*' --filter !test-browser run test",
"test:ci:cache": "CI=true pnpm -r --reporter-hide-prefix --stream --sequential --filter '@vitest/test-*' --filter !test-browser --filter !test-dts-config --filter !test-dts-fixture --filter !test-dts-playwright --filter !test-ui --filter !test-cache run test --experimental.fsModuleCache",
"test:examples": "CI=true pnpm -r --reporter-hide-prefix --stream --filter '@vitest/example-*' run test",
"test:ecosystem-ci": "ECOSYSTEM_CI=true pnpm test:ci",
"typebuild": "tsx ./scripts/explain-types.ts",

View File

@ -50,15 +50,23 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider<ResolvedCover
})
}
onFileTransform(sourceCode: string, id: string, pluginCtx: any): { code: string; map: any } | undefined {
requiresTransform(id: string): boolean {
// Istanbul/babel cannot instrument CSS - e.g. Vue imports end up here.
// File extension itself is .vue, but it contains CSS.
// e.g. "Example.vue?vue&type=style&index=0&scoped=f7f04e08&lang.css"
if (isCSSRequest(id)) {
return
return false
}
if (!this.isIncluded(removeQueryParameters(id))) {
return false
}
return true
}
onFileTransform(sourceCode: string, id: string, pluginCtx: Vite.Rollup.TransformPluginContext): { code: string; map: any } | undefined {
if (!this.requiresTransform(id)) {
return
}

View File

@ -0,0 +1,508 @@
import type { DevEnvironment, FetchResult } from 'vite'
import type { Vitest } from '../core'
import type { VitestResolver } from '../resolver'
import type { ResolvedConfig } from '../types/config'
import fs, { existsSync, mkdirSync, readFileSync } from 'node:fs'
import { readFile, rename, rm, stat, unlink, writeFile } from 'node:fs/promises'
import { parse, stringify } from 'flatted'
import { dirname, join } from 'pathe'
import c from 'tinyrainbow'
import { searchForWorkspaceRoot } from 'vite'
import { createDebugger } from '../../utils/debugger'
import { hash } from '../hash'
const debugFs = createDebugger('vitest:cache:fs')
const debugMemory = createDebugger('vitest:cache:memory')
const cacheComment = '\n//# vitestCache='
const cacheCommentLength = cacheComment.length
const METADATA_FILE = '_metadata.json'
/**
* @experimental
*/
export class FileSystemModuleCache {
/**
* Even though it's possible to override the folder of project's caches
* We still keep a single metadata file for all projects because
* - they can reference files between each other
* - lockfile changes are reflected for the whole workspace, not just for a single project
*/
private rootCache: string
private metadataFilePath: string
private version = '1.0.0-beta.1'
private fsCacheRoots = new WeakMap<ResolvedConfig, string>()
private fsEnvironmentHashMap = new WeakMap<DevEnvironment, string>()
private fsCacheKeyGenerators = new Set<CacheKeyIdGenerator>()
// this exists only to avoid the perf. cost of reading a file and generating a hash again
// surprisingly, on some machines this has negligible effect
private fsCacheKeys = new WeakMap<
DevEnvironment,
// Map<id, tmp | null>
Map<string, string | null>
>()
constructor(private vitest: Vitest) {
const workspaceRoot = searchForWorkspaceRoot(vitest.vite.config.root)
this.rootCache = vitest.config.experimental.fsModuleCachePath
|| join(workspaceRoot, 'node_modules', '.experimental-vitest-cache')
this.metadataFilePath = join(this.rootCache, METADATA_FILE)
}
public defineCacheKeyGenerator(callback: CacheKeyIdGenerator): void {
this.fsCacheKeyGenerators.add(callback)
}
async clearCache(log = true): Promise<void> {
const fsCachePaths = this.vitest.projects.map((r) => {
return r.config.experimental.fsModuleCachePath || this.rootCache
})
const uniquePaths = Array.from(new Set(fsCachePaths))
await Promise.all(
uniquePaths.map(directory => rm(directory, { force: true, recursive: true })),
)
if (log) {
this.vitest.logger.log(`[cache] cleared fs module cache at ${uniquePaths.join(', ')}`)
}
}
async getCachedModule(cachedFilePath: string): Promise<
CachedInlineModuleMeta
| Extract<FetchResult, { externalize: string }>
| undefined
> {
if (!existsSync(cachedFilePath)) {
debugFs?.(`${c.red('[empty]')} ${cachedFilePath} doesn't exist, transforming by vite instead`)
return
}
const code = await readFile(cachedFilePath, 'utf-8')
const matchIndex = code.lastIndexOf(cacheComment)
if (matchIndex === -1) {
debugFs?.(`${c.red('[empty]')} ${cachedFilePath} exists, but doesn't have a ${cacheComment} comment, transforming by vite instead`)
return
}
const meta = this.fromBase64(code.slice(matchIndex + cacheCommentLength))
if (meta.externalize) {
debugFs?.(`${c.green('[read]')} ${meta.externalize} is externalized inside ${cachedFilePath}`)
return { externalize: meta.externalize, type: meta.type }
}
debugFs?.(`${c.green('[read]')} ${meta.id} is cached in ${cachedFilePath}`)
return {
id: meta.id,
url: meta.url,
file: meta.file,
code,
importers: meta.importers,
mappings: meta.mappings,
}
}
async saveCachedModule<T extends FetchResult>(
cachedFilePath: string,
fetchResult: T,
importers: string[] = [],
mappings: boolean = false,
): Promise<void> {
if ('externalize' in fetchResult) {
debugFs?.(`${c.yellow('[write]')} ${fetchResult.externalize} is externalized inside ${cachedFilePath}`)
await atomicWriteFile(cachedFilePath, `${cacheComment}${this.toBase64(fetchResult)}`)
}
else if ('code' in fetchResult) {
const result = {
file: fetchResult.file,
id: fetchResult.id,
url: fetchResult.url,
importers,
mappings,
} satisfies Omit<FetchResult, 'code' | 'invalidate'>
debugFs?.(`${c.yellow('[write]')} ${fetchResult.id} is cached in ${cachedFilePath}`)
await atomicWriteFile(cachedFilePath, `${fetchResult.code}${cacheComment}${this.toBase64(result)}`)
}
}
private toBase64(obj: unknown) {
const json = stringify(obj)
return Buffer.from(json).toString('base64')
}
private fromBase64(obj: string) {
const json = Buffer.from(obj, 'base64').toString('utf-8')
return parse(json)
}
invalidateCachePath(
environment: DevEnvironment,
id: string,
): void {
debugFs?.(`cache for ${id} in ${environment.name} environment is invalidated`)
this.fsCacheKeys.get(environment)?.delete(id)
}
invalidateAllCachePaths(environment: DevEnvironment): void {
debugFs?.(`the ${environment.name} environment cache is invalidated`)
this.fsCacheKeys.get(environment)?.clear()
}
getMemoryCachePath(
environment: DevEnvironment,
id: string,
): string | null | undefined {
const result = this.fsCacheKeys.get(environment)?.get(id)
if (result != null) {
debugMemory?.(`${c.green('[read]')} ${id} was cached in ${result}`)
}
else if (result === null) {
debugMemory?.(`${c.green('[read]')} ${id} was bailed out`)
}
return result
}
generateCachePath(
vitestConfig: ResolvedConfig,
environment: DevEnvironment,
resolver: VitestResolver,
id: string,
fileContent: string,
): string | null {
let hashString = ''
// bail out if file has import.meta.glob because it depends on other files
// TODO: figure out a way to still support it
if (fileContent.includes('import.meta.glob(')) {
this.saveMemoryCache(environment, id, null)
debugMemory?.(`${c.yellow('[write]')} ${id} was bailed out`)
return null
}
for (const generator of this.fsCacheKeyGenerators) {
const result = generator({ environment, id, sourceCode: fileContent })
if (typeof result === 'string') {
hashString += result
}
if (result === false) {
this.saveMemoryCache(environment, id, null)
debugMemory?.(`${c.yellow('[write]')} ${id} was bailed out by a custom generator`)
return null
}
}
const config = environment.config
// coverage provider is dynamic, so we also clear the whole cache if
// vitest.enableCoverage/vitest.disableCoverage is called
const coverageAffectsCache = String(this.vitest.config.coverage.enabled && this.vitest.coverageProvider?.requiresTransform?.(id))
let cacheConfig = this.fsEnvironmentHashMap.get(environment)
if (!cacheConfig) {
cacheConfig = JSON.stringify(
{
root: config.root,
// at the moment, Vitest always forces base to be /
base: config.base,
mode: config.mode,
consumer: config.consumer,
resolve: config.resolve,
// plugins can have different options, so this is not the best key,
// but we canot access the options because there is no standard API for it
plugins: config.plugins.map(p => p.name),
// in case local plugins change
// configFileDependencies also includes configFile
configFileDependencies: config.configFileDependencies.map(file => tryReadFileSync(file)),
environment: environment.name,
// this affects Vitest CSS plugin
css: vitestConfig.css,
// this affect externalization
resolver: {
inline: resolver.options.inline,
external: resolver.options.external,
inlineFiles: resolver.options.inlineFiles,
moduleDirectories: resolver.options.moduleDirectories,
},
},
(_, value) => {
if (typeof value === 'function' || value instanceof RegExp) {
return value.toString()
}
return value
},
)
this.fsEnvironmentHashMap.set(environment, cacheConfig)
}
hashString += id
+ fileContent
+ (process.env.NODE_ENV ?? '')
+ this.version
+ cacheConfig
+ coverageAffectsCache
const cacheKey = hash('sha1', hashString, 'hex')
let cacheRoot = this.fsCacheRoots.get(vitestConfig)
if (cacheRoot == null) {
cacheRoot = vitestConfig.experimental.fsModuleCachePath || this.rootCache
if (!existsSync(cacheRoot)) {
mkdirSync(cacheRoot, { recursive: true })
}
}
const fsResultPath = join(cacheRoot, cacheKey)
debugMemory?.(`${c.yellow('[write]')} ${id} generated a cache in ${fsResultPath}`)
this.saveMemoryCache(environment, id, fsResultPath)
return fsResultPath
}
private saveMemoryCache(environment: DevEnvironment, id: string, cache: string | null) {
let environmentKeys = this.fsCacheKeys.get(environment)
if (!environmentKeys) {
environmentKeys = new Map()
this.fsCacheKeys.set(environment, environmentKeys)
}
environmentKeys.set(id, cache)
}
private async readMetadata(): Promise<{ lockfileHash: string } | undefined> {
// metadata is shared between every projects in the workspace, so we ignore project's fsModuleCachePath
if (!existsSync(this.metadataFilePath)) {
return undefined
}
try {
const content = await readFile(this.metadataFilePath, 'utf-8')
return JSON.parse(content)
}
catch {}
}
// before vitest starts running tests, we check that the lockfile wasn't updated
// if it was, we nuke the previous cache in case a custom plugin was updated
// or a new version of vite/vitest is installed
// for the same reason we also cache config file content, but that won't catch changes made in external plugins
public async ensureCacheIntegrity(): Promise<void> {
const enabled = [
this.vitest.getRootProject(),
...this.vitest.projects,
].some(p => p.config.experimental.fsModuleCache)
if (!enabled) {
return
}
const metadata = await this.readMetadata()
const currentLockfileHash = getLockfileHash(this.vitest.vite.config.root)
// no metadata found, just store a new one, don't reset the cache
if (!metadata) {
if (!existsSync(this.rootCache)) {
mkdirSync(this.rootCache, { recursive: true })
}
debugFs?.(`fs metadata file was created with hash ${currentLockfileHash}`)
await writeFile(
this.metadataFilePath,
JSON.stringify({ lockfileHash: currentLockfileHash }, null, 2),
'utf-8',
)
return
}
// if lockfile didn't change, don't do anything
if (metadata.lockfileHash === currentLockfileHash) {
return
}
// lockfile changed, let's clear all caches
await this.clearCache(false)
this.vitest.vite.config.logger.info(
`fs cache was cleared because lockfile has changed`,
{
timestamp: true,
environment: c.yellow('[vitest]'),
},
)
debugFs?.(`fs cache was cleared because lockfile has changed`)
}
}
/**
* Performs an atomic write operation using the write-then-rename pattern.
*
* Why we need this:
* - Ensures file integrity by never leaving partially written files on disk
* - Prevents other processes from reading incomplete data during writes
* - Particularly important for test files where incomplete writes could cause test failures
*
* The implementation writes to a temporary file first, then renames it to the target path.
* This rename operation is atomic on most filesystems (including POSIX-compliant ones),
* guaranteeing that other processes will only ever see the complete file.
*
* Added in https://github.com/vitest-dev/vitest/pull/7531
*/
async function atomicWriteFile(realFilePath: string, data: string): Promise<void> {
const dir = dirname(realFilePath)
const tmpFilePath = join(dir, `.tmp-${Date.now()}-${Math.random().toString(36).slice(2)}`)
try {
await writeFile(tmpFilePath, data, 'utf-8')
await rename(tmpFilePath, realFilePath)
}
finally {
try {
if (await stat(tmpFilePath)) {
await unlink(tmpFilePath)
}
}
catch {}
}
}
export interface CachedInlineModuleMeta {
url: string
id: string
file: string | null
code: string
importers: string[]
mappings: boolean
}
/**
* Generate a unique cache identifier.
*
* Return `false` to disable caching of the file.
* @experimental
*/
export interface CacheKeyIdGenerator {
(context: CacheKeyIdGeneratorContext): string | undefined | null | false
}
/**
* @experimental
*/
export interface CacheKeyIdGeneratorContext {
environment: DevEnvironment
id: string
sourceCode: string
}
// lockfile hash resolution taken from vite
// since this is experimental, we don't ask to expose it
const lockfileFormats = [
{
path: 'node_modules/.package-lock.json',
checkPatchesDir: 'patches',
manager: 'npm',
},
{
// Yarn non-PnP
path: 'node_modules/.yarn-state.yml',
checkPatchesDir: false,
manager: 'yarn',
},
{
// Yarn v3+ PnP
path: '.pnp.cjs',
checkPatchesDir: '.yarn/patches',
manager: 'yarn',
},
{
// Yarn v2 PnP
path: '.pnp.js',
checkPatchesDir: '.yarn/patches',
manager: 'yarn',
},
{
// yarn 1
path: 'node_modules/.yarn-integrity',
checkPatchesDir: 'patches',
manager: 'yarn',
},
{
path: 'node_modules/.pnpm/lock.yaml',
// Included in lockfile
checkPatchesDir: false,
manager: 'pnpm',
},
{
path: '.rush/temp/shrinkwrap-deps.json',
// Included in lockfile
checkPatchesDir: false,
manager: 'pnpm',
},
{
path: 'bun.lock',
checkPatchesDir: 'patches',
manager: 'bun',
},
{
path: 'bun.lockb',
checkPatchesDir: 'patches',
manager: 'bun',
},
].sort((_, { manager }) => {
return process.env.npm_config_user_agent?.startsWith(manager) ? 1 : -1
})
const lockfilePaths = lockfileFormats.map(l => l.path)
function getLockfileHash(root: string): string {
const lockfilePath = lookupFile(root, lockfilePaths)
let content = lockfilePath ? fs.readFileSync(lockfilePath, 'utf-8') : ''
if (lockfilePath) {
const normalizedLockfilePath = lockfilePath.replaceAll('\\', '/')
const lockfileFormat = lockfileFormats.find(f =>
normalizedLockfilePath.endsWith(f.path),
)!
if (lockfileFormat.checkPatchesDir) {
// Default of https://github.com/ds300/patch-package
const baseDir = lockfilePath.slice(0, -lockfileFormat.path.length)
const fullPath = join(
baseDir,
lockfileFormat.checkPatchesDir as string,
)
const stat = tryStatSync(fullPath)
if (stat?.isDirectory()) {
content += stat.mtimeMs.toString()
}
}
}
return hash('sha256', content, 'hex').substring(0, 8).padEnd(8, '_')
}
function lookupFile(
dir: string,
fileNames: string[],
): string | undefined {
while (dir) {
for (const fileName of fileNames) {
const fullPath = join(dir, fileName)
if (tryStatSync(fullPath)?.isFile()) {
return fullPath
}
}
const parentDir = dirname(dir)
if (parentDir === dir) {
return
}
dir = parentDir
}
}
function tryReadFileSync(file: string): string {
try {
return readFileSync(file, 'utf-8')
}
catch {
return ''
}
}
function tryStatSync(file: string): fs.Stats | undefined {
try {
// The "throwIfNoEntry" is a performance optimization for cases where the file does not exist
return fs.statSync(file, { throwIfNoEntry: false })
}
catch {
// Ignore errors
}
}

View File

@ -1,3 +1,4 @@
import type { Logger } from '../logger'
import type { SuiteResultCache } from './results'
import { slash } from '@vitest/utils/helpers'
import { resolve } from 'pathe'
@ -9,8 +10,8 @@ export class VitestCache {
results: ResultsCache
stats: FilesStatsCache = new FilesStatsCache()
constructor(version: string) {
this.results = new ResultsCache(version)
constructor(logger: Logger) {
this.results = new ResultsCache(logger)
}
getFileTestResults(key: string): SuiteResultCache | undefined {

View File

@ -1,7 +1,10 @@
import type { File } from '@vitest/runner'
import type { Logger } from '../logger'
import type { ResolvedConfig } from '../types/config'
import fs from 'node:fs'
import fs, { existsSync } from 'node:fs'
import { rm } from 'node:fs/promises'
import { dirname, relative, resolve } from 'pathe'
import { Vitest } from '../core'
export interface SuiteResultCache {
failed: boolean
@ -15,8 +18,8 @@ export class ResultsCache {
private version: string
private root = '/'
constructor(version: string) {
this.version = version
constructor(private logger: Logger) {
this.version = Vitest.version
}
public getCachePath(): string | null {
@ -34,6 +37,13 @@ export class ResultsCache {
return this.cache.get(key)
}
async clearCache(): Promise<void> {
if (this.cachePath && existsSync(this.cachePath)) {
await rm(this.cachePath, { force: true, recursive: true })
this.logger.log('[cache] cleared results cache at', this.cachePath)
}
}
async readFromCache(): Promise<void> {
if (!this.cachePath) {
return

View File

@ -288,6 +288,10 @@ function normalizeCliOptions(cliFilters: string[], argv: CliOptions): CliOptions
if (typeof argv.typecheck?.only === 'boolean') {
argv.typecheck.enabled ??= true
}
if (argv.clearCache) {
argv.watch = false
argv.run = true
}
return argv
}

View File

@ -91,7 +91,10 @@ export async function startVitest(
})
try {
if (ctx.config.mergeReports) {
if (ctx.config.clearCache) {
await ctx.experimental_clearCache()
}
else if (ctx.config.mergeReports) {
await ctx.mergeReports()
}
else if (ctx.config.standalone) {

View File

@ -774,8 +774,21 @@ export const cliOptionsConfig: VitestCLIOptions = {
return value
},
},
clearCache: {
description: 'Delete all Vitest caches, including `experimental.fsModuleCache`, without running any tests. This will reduce the performance in the subsequent test run.',
},
experimental: null,
experimental: {
description: 'Experimental features.',
argument: '<features>',
subcommands: {
fsModuleCache: {
description: 'Enable caching of modules on the file system between reruns.',
},
fsModuleCachePath: null,
openTelemetry: null,
},
},
// disable CLI options
cliExclude: null,
server: null,

View File

@ -806,6 +806,12 @@ export function resolveConfig(
)
resolved.experimental.openTelemetry.sdkPath = pathToFileURL(sdkPath).toString()
}
if (resolved.experimental.fsModuleCachePath) {
resolved.experimental.fsModuleCachePath = resolve(
resolved.root,
resolved.experimental.fsModuleCachePath,
)
}
return resolved
}

View File

@ -130,5 +130,8 @@ export function serializeConfig(project: TestProject): SerializedConfig {
serializedDefines: config.browser.enabled
? ''
: project._serializedDefines || '',
experimental: {
fsModuleCache: config.experimental.fsModuleCache ?? false,
},
}
}

View File

@ -28,6 +28,7 @@ import { Traces } from '../utils/traces'
import { astCollectTests, createFailedFileTask } from './ast-collect'
import { BrowserSessions } from './browser/sessions'
import { VitestCache } from './cache'
import { FileSystemModuleCache } from './cache/fsModuleCache'
import { resolveConfig } from './config/resolveConfig'
import { getCoverageProvider } from './coverage'
import { createFetchModuleFunction } from './environments/fetchModule'
@ -110,6 +111,7 @@ export class Vitest {
/** @internal */ _testRun: TestRun = undefined!
/** @internal */ _resolver!: VitestResolver
/** @internal */ _fetcher!: VitestFetchFunction
/** @internal */ _fsCache!: FileSystemModuleCache
/** @internal */ _tmpDir = join(tmpdir(), nanoid())
/** @internal */ _traces!: Traces
@ -210,7 +212,7 @@ export class Vitest {
this._state = new StateManager({
onUnhandledError: resolved.onUnhandledError,
})
this._cache = new VitestCache(this.version)
this._cache = new VitestCache(this.logger)
this._snapshot = new SnapshotManager({ ...resolved.snapshotOptions })
this._testRun = new TestRun(this)
const otelSdkPath = resolved.experimental.openTelemetry?.sdkPath
@ -225,14 +227,13 @@ export class Vitest {
}
this._resolver = new VitestResolver(server.config.cacheDir, resolved)
this._fsCache = new FileSystemModuleCache(this)
this._fetcher = createFetchModuleFunction(
this._resolver,
this._config,
this._fsCache,
this._traces,
this._tmpDir,
{
dumpFolder: this.config.dumpDir,
readFromDump: this.config.server.debug?.load ?? process.env.VITEST_DEBUG_LOAD_DUMP != null,
},
)
const environment = server.environments.__vitest__
this.runner = new ServerModuleRunner(
@ -280,6 +281,10 @@ export class Vitest {
project,
vitest: this,
injectTestProjects: this.injectTestProject,
/**
* @experimental
*/
experimental_defineCacheKeyGenerator: callback => this._fsCache.defineCacheKeyGenerator(callback),
}))
}))
@ -315,6 +320,8 @@ export class Vitest {
? await createBenchmarkReporters(toArray(resolved.benchmark?.reporters), this.runner)
: await createReporters(resolved.reporters, this)
await this._fsCache.ensureCacheIntegrity()
await Promise.all([
...this._onSetServer.map(fn => fn()),
this._traces.waitInit(),
@ -334,11 +341,32 @@ export class Vitest {
this.configOverride.coverage!.enabled = true
await this.createCoverageProvider()
await this.coverageProvider?.onEnabled?.()
// onFileTransform is the only thing that affects hash
if (this.coverageProvider?.onFileTransform) {
this.clearAllCachePaths()
}
}
public disableCoverage(): void {
this.configOverride.coverage ??= {} as any
this.configOverride.coverage!.enabled = false
// onFileTransform is the only thing that affects hash
if (this.coverageProvider?.onFileTransform) {
this.clearAllCachePaths()
}
}
private clearAllCachePaths() {
this.projects.forEach(({ vite, browser }) => {
const environments = [
...Object.values(vite.environments),
...Object.values(browser?.vite.environments || {}),
]
environments.forEach(environment =>
this._fsCache.invalidateAllCachePaths(environment),
)
})
}
private _coverageOverrideCache = new WeakMap<ResolvedCoverageOptions, ResolvedCoverageOptions>()
@ -494,6 +522,15 @@ export class Vitest {
return this._coverageProvider
}
/**
* Deletes all Vitest caches, including `experimental.fsModuleCache`.
* @experimental
*/
public async experimental_clearCache(): Promise<void> {
await this.cache.results.clearCache()
await this._fsCache.clearCache()
}
/**
* Merge reports from multiple runs located in the specified directory (value from `--merge-reports` if not specified).
*/
@ -1182,9 +1219,17 @@ export class Vitest {
...Object.values(browser?.vite.environments || {}),
]
environments.forEach(({ moduleGraph }) => {
environments.forEach((environment) => {
const { moduleGraph } = environment
const modules = moduleGraph.getModulesByFile(filepath)
modules?.forEach(module => moduleGraph.invalidateModule(module))
if (!modules) {
return
}
modules.forEach((module) => {
moduleGraph.invalidateModule(module)
this._fsCache.invalidateCachePath(environment, module.id!)
})
})
})
}

View File

@ -1,23 +1,323 @@
import type { Span } from '@opentelemetry/api'
import type { DevEnvironment, FetchResult, Rollup, TransformResult } from 'vite'
import type { DevEnvironment, EnvironmentModuleNode, FetchResult, Rollup, TransformResult } from 'vite'
import type { FetchFunctionOptions } from 'vite/module-runner'
import type { FetchCachedFileSystemResult } from '../../types/general'
import type { OTELCarrier, Traces } from '../../utils/traces'
import type { FileSystemModuleCache } from '../cache/fsModuleCache'
import type { VitestResolver } from '../resolver'
import type { ResolvedConfig } from '../types/config'
import { existsSync, mkdirSync } from 'node:fs'
import { readFile, rename, stat, unlink, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { isExternalUrl, nanoid, unwrapId } from '@vitest/utils/helpers'
import { dirname, join, resolve } from 'pathe'
import { readFile } from 'node:fs/promises'
import { isExternalUrl, unwrapId } from '@vitest/utils/helpers'
import { join } from 'pathe'
import { fetchModule } from 'vite'
import { hash } from '../hash'
const created = new Set()
const promises = new Map<string, Promise<void>>()
const saveCachePromises = new Map<string, Promise<FetchResult>>()
const readFilePromises = new Map<string, Promise<string | null>>()
interface DumpOptions {
dumpFolder?: string
readFromDump?: boolean
class ModuleFetcher {
private tmpDirectories = new Set<string>()
private fsCacheEnabled: boolean
constructor(
private resolver: VitestResolver,
private config: ResolvedConfig,
private fsCache: FileSystemModuleCache,
private traces: Traces,
private tmpProjectDir: string,
) {
this.fsCacheEnabled = config.experimental?.fsModuleCache === true
}
async fetch(
trace: Span,
url: string,
importer: string | undefined,
environment: DevEnvironment,
makeTmpCopies?: boolean,
options?: FetchFunctionOptions,
): Promise<FetchResult | FetchCachedFileSystemResult> {
if (url.startsWith('data:')) {
trace.setAttribute('vitest.module.external', url)
return { externalize: url, type: 'builtin' }
}
if (url === '/@vite/client' || url === '@vite/client') {
trace.setAttribute('vitest.module.external', url)
return { externalize: '/@vite/client', type: 'module' }
}
const isFileUrl = url.startsWith('file://')
if (isExternalUrl(url) && !isFileUrl) {
trace.setAttribute('vitest.module.external', url)
return { externalize: url, type: 'network' }
}
const moduleGraphModule = await environment.moduleGraph.ensureEntryFromUrl(unwrapId(url))
const cached = !!moduleGraphModule.transformResult
if (moduleGraphModule.file) {
trace.setAttribute('code.file.path', moduleGraphModule.file)
}
if (options?.cached && cached) {
return { cache: true }
}
const cachePath = await this.getCachePath(
environment,
moduleGraphModule,
)
// full fs caching is disabled, but we still want to keep tmp files if makeTmpCopies is enabled
// this is primarily used by the forks pool to avoid using process.send(bigBuffer)
if (cachePath == null) {
const result = await this.fetchAndProcess(environment, url, importer, moduleGraphModule, options)
this.recordResult(trace, result)
if (!makeTmpCopies || !('code' in result)) {
return result
}
const transformResult = moduleGraphModule.transformResult
const tmpPath = transformResult && Reflect.get(transformResult, '_vitest_tmp')
if (typeof tmpPath === 'string') {
return getCachedResult(result, tmpPath)
}
const tmpDir = join(this.tmpProjectDir, environment.name)
if (!this.tmpDirectories.has(tmpDir)) {
if (!existsSync(tmpDir)) {
mkdirSync(tmpDir, { recursive: true })
}
this.tmpDirectories.add(tmpDir)
}
const tmpFile = join(tmpDir, hash('sha1', result.id, 'hex'))
return this.cacheResult(result, tmpFile).then((result) => {
if (transformResult) {
Reflect.set(transformResult, '_vitest_tmp', tmpFile)
}
return result
})
}
if (saveCachePromises.has(cachePath)) {
return saveCachePromises.get(cachePath)!.then((result) => {
this.recordResult(trace, result)
return result
})
}
const cachedModule = await this.getCachedModule(cachePath, environment, moduleGraphModule)
if (cachedModule) {
this.recordResult(trace, cachedModule)
return cachedModule
}
const result = await this.fetchAndProcess(environment, url, importer, moduleGraphModule, options)
const importers = this.getSerializedDependencies(moduleGraphModule)
const map = moduleGraphModule.transformResult?.map
const mappings = map && !('version' in map) && map.mappings === ''
return this.cacheResult(result, cachePath, importers, !!mappings)
}
private getSerializedDependencies(node: EnvironmentModuleNode): string[] {
const dependencies: string[] = []
node.importers.forEach((importer) => {
if (importer.id) {
dependencies.push(importer.id)
}
})
return dependencies
}
private recordResult(trace: Span, result: FetchResult | FetchCachedFileSystemResult): void {
if ('externalize' in result) {
trace.setAttributes({
'vitest.module.external': result.externalize,
'vitest.fetched_module.type': result.type,
})
}
if ('id' in result) {
trace.setAttributes({
'vitest.fetched_module.invalidate': result.invalidate,
'vitest.fetched_module.id': result.id,
'vitest.fetched_module.url': result.url,
'vitest.fetched_module.cache': false,
})
if (result.file) {
trace.setAttribute('code.file.path', result.file)
}
}
if ('code' in result) {
trace.setAttribute('vitest.fetched_module.code_length', result.code.length)
}
}
private async getCachePath(environment: DevEnvironment, moduleGraphModule: EnvironmentModuleNode): Promise<null | string> {
if (!this.fsCacheEnabled) {
return null
}
const moduleId = moduleGraphModule.id!
const memoryCacheKey = this.fsCache.getMemoryCachePath(environment, moduleId)
// undefined means there is no key in memory
// null means the file should not be cached
if (memoryCacheKey !== undefined) {
return memoryCacheKey
}
const fileContent = await this.readFileContentToCache(environment, moduleGraphModule)
return this.fsCache.generateCachePath(
this.config,
environment,
this.resolver,
moduleGraphModule.id!,
fileContent,
)
}
private async readFileContentToCache(
environment: DevEnvironment,
moduleGraphModule: EnvironmentModuleNode,
): Promise<string> {
if (
moduleGraphModule.file
// \x00 is a virtual file convention
&& !moduleGraphModule.file.startsWith('\x00')
&& !moduleGraphModule.file.startsWith('virtual:')
) {
const result = await this.readFileConcurrently(moduleGraphModule.file)
if (result != null) {
return result
}
}
const loadResult = await environment.pluginContainer.load(moduleGraphModule.id!)
if (typeof loadResult === 'string') {
return loadResult
}
if (loadResult != null) {
return loadResult.code
}
return ''
}
private async getCachedModule(
cachePath: string,
environment: DevEnvironment,
moduleGraphModule: EnvironmentModuleNode,
): Promise<FetchResult | FetchCachedFileSystemResult | undefined> {
const cachedModule = await this.fsCache.getCachedModule(cachePath)
if (cachedModule && 'code' in cachedModule) {
// keep the module graph in sync
if (!moduleGraphModule.transformResult) {
let map: Rollup.SourceMap | null | { mappings: '' } = extractSourceMap(cachedModule.code)
if (map && cachedModule.file) {
map.file = cachedModule.file
}
// mappings is a special source map identifier in rollup
if (!map && cachedModule.mappings) {
map = { mappings: '' }
}
moduleGraphModule.transformResult = {
code: cachedModule.code,
map,
ssr: true,
}
// we populate the module graph to make the watch mode work because it relies on importers
cachedModule.importers.forEach((importer) => {
const environmentNode = environment.moduleGraph.getModuleById(importer)
if (environmentNode) {
moduleGraphModule.importers.add(environmentNode)
}
})
}
return {
cached: true as const,
file: cachedModule.file,
id: cachedModule.id,
tmp: cachePath,
url: cachedModule.url,
invalidate: false,
}
}
return cachedModule
}
private async fetchAndProcess(
environment: DevEnvironment,
url: string,
importer: string | undefined,
moduleGraphModule: EnvironmentModuleNode,
options?: FetchFunctionOptions,
): Promise<FetchResult> {
const externalize = await this.resolver.shouldExternalize(moduleGraphModule.id!)
if (externalize) {
return { externalize, type: 'module' }
}
const moduleRunnerModule = await fetchModule(
environment,
url,
importer,
{
...options,
inlineSourceMap: false,
},
).catch(handleRollupError)
return processResultSource(environment, moduleRunnerModule)
}
private async cacheResult(
result: FetchResult,
cachePath: string,
importers: string[] = [],
mappings = false,
): Promise<FetchResult | FetchCachedFileSystemResult> {
const returnResult = 'code' in result
? getCachedResult(result, cachePath)
: result
if (saveCachePromises.has(cachePath)) {
await saveCachePromises.get(cachePath)
return returnResult
}
const savePromise = this.fsCache
.saveCachedModule(cachePath, result, importers, mappings)
.then(() => result)
.finally(() => {
saveCachePromises.delete(cachePath)
})
saveCachePromises.set(cachePath, savePromise)
await savePromise
return returnResult
}
private readFileConcurrently(file: string): Promise<string | null> {
if (!readFilePromises.has(file)) {
readFilePromises.set(
file,
// virtual file can have a "file" property
readFile(file, 'utf-8').catch(() => null).finally(() => {
readFilePromises.delete(file)
}),
)
}
return readFilePromises.get(file)!
}
}
export interface VitestFetchFunction {
@ -25,7 +325,7 @@ export interface VitestFetchFunction {
url: string,
importer: string | undefined,
environment: DevEnvironment,
cacheFs: boolean,
cacheFs?: boolean,
options?: FetchFunctionOptions,
otelCarrier?: OTELCarrier
): Promise<FetchResult | FetchCachedFileSystemResult>
@ -33,172 +333,13 @@ export interface VitestFetchFunction {
export function createFetchModuleFunction(
resolver: VitestResolver,
config: ResolvedConfig,
fsCache: FileSystemModuleCache,
traces: Traces,
tmpDir: string = join(tmpdir(), nanoid()),
dump?: DumpOptions,
tmpProjectDir: string,
): VitestFetchFunction {
const fetcher = async (
fetcherSpan: Span,
url: string,
importer: string | undefined,
environment: DevEnvironment,
cacheFs: boolean,
options?: FetchFunctionOptions,
): Promise<FetchResult | FetchCachedFileSystemResult> => {
// We are copy pasting Vite's externalization logic from `fetchModule` because
// we instead rely on our own `shouldExternalize` method because Vite
// doesn't support `resolve.external` in non SSR environments (jsdom/happy-dom)
if (url.startsWith('data:')) {
fetcherSpan.setAttribute('vitest.module.external', url)
return { externalize: url, type: 'builtin' }
}
if (url === '/@vite/client' || url === '@vite/client') {
fetcherSpan.setAttribute('vitest.module.external', url)
// this will be stubbed
return { externalize: '/@vite/client', type: 'module' }
}
const isFileUrl = url.startsWith('file://')
if (isExternalUrl(url) && !isFileUrl) {
fetcherSpan.setAttribute('vitest.module.external', url)
return { externalize: url, type: 'network' }
}
// Vite does the same in `fetchModule`, but we want to externalize modules ourselves,
// so we do this first to resolve the module and check its `id`. The next call of
// `ensureEntryFromUrl` inside `fetchModule` is cached and should take no time
// This also makes it so externalized modules are inside the module graph.
const moduleGraphModule = await environment.moduleGraph.ensureEntryFromUrl(unwrapId(url))
const cached = !!moduleGraphModule.transformResult
if (moduleGraphModule.file) {
fetcherSpan.setAttribute('code.file.path', moduleGraphModule.file)
}
// if url is already cached, we can just confirm it's also cached on the server
if (options?.cached && cached) {
return { cache: true }
}
if (moduleGraphModule.id) {
const id = moduleGraphModule.id
const externalize = await resolver.shouldExternalize(id)
if (externalize) {
fetcherSpan.setAttribute('vitest.module.external', externalize)
return { externalize, type: 'module' }
}
}
fetcherSpan.setAttribute('vitest.module.external', false)
let moduleRunnerModule: FetchResult | undefined
if (dump?.dumpFolder && dump.readFromDump) {
const path = resolve(dump?.dumpFolder, url.replace(/[^\w+]/g, '-'))
if (existsSync(path)) {
const code = await readFile(path, 'utf-8')
const matchIndex = code.lastIndexOf('\n//')
if (matchIndex !== -1) {
const { id, file } = JSON.parse(code.slice(matchIndex + 4))
moduleRunnerModule = {
code,
id,
url,
file,
invalidate: false,
}
}
}
}
if (!moduleRunnerModule) {
moduleRunnerModule = await fetchModule(
environment,
url,
importer,
{
...options,
inlineSourceMap: false,
},
).catch(handleRollupError)
}
if ('id' in moduleRunnerModule) {
fetcherSpan.setAttributes({
'vitest.fetched_module.invalidate': moduleRunnerModule.invalidate,
'vitest.fetched_module.code_length': moduleRunnerModule.code.length,
'vitest.fetched_module.id': moduleRunnerModule.id,
'vitest.fetched_module.url': moduleRunnerModule.url,
'vitest.fetched_module.cache': false,
})
if (moduleRunnerModule.file) {
fetcherSpan.setAttribute('code.file.path', moduleRunnerModule.file)
}
}
else if ('cache' in moduleRunnerModule) {
fetcherSpan.setAttribute('vitest.fetched_module.cache', moduleRunnerModule.cache)
}
else {
fetcherSpan.setAttribute('vitest.fetched_module.type', moduleRunnerModule.type)
fetcherSpan.setAttribute('vitest.fetched_module.external', moduleRunnerModule.externalize)
}
const result = processResultSource(environment, moduleRunnerModule)
if (dump?.dumpFolder && 'code' in result) {
const path = resolve(dump?.dumpFolder, result.url.replace(/[^\w+]/g, '-'))
await writeFile(path, `${result.code}\n// ${JSON.stringify({ id: result.id, file: result.file })}`, 'utf-8')
}
if (!cacheFs || !('code' in result)) {
return result
}
const code = result.code
const transformResult = result.transformResult!
if (!transformResult) {
throw new Error(`"transformResult" in not defined. This is a bug in Vitest.`)
}
// to avoid serialising large chunks of code,
// we store them in a tmp file and read in the test thread
if ('_vitestTmp' in transformResult) {
return getCachedResult(result, Reflect.get(transformResult as any, '_vitestTmp'))
}
const dir = join(tmpDir, environment.name)
const name = hash('sha1', result.id, 'hex')
const tmp = join(dir, name)
if (!created.has(dir)) {
mkdirSync(dir, { recursive: true })
created.add(dir)
}
if (promises.has(tmp)) {
await promises.get(tmp)
return getCachedResult(result, tmp)
}
promises.set(
tmp,
atomicWriteFile(tmp, code)
// Fallback to non-atomic write for windows case where file already exists:
.catch(() => writeFile(tmp, code, 'utf-8'))
.finally(() => {
Reflect.set(transformResult, '_vitestTmp', tmp)
promises.delete(tmp)
}),
)
await promises.get(tmp)
return getCachedResult(result, tmp)
}
return async (
url,
importer,
environment,
cacheFs,
options,
otelCarrier,
) => {
const fetcher = new ModuleFetcher(resolver, config, fsCache, traces, tmpProjectDir)
return async (url, importer, environment, cacheFs, options, otelCarrier) => {
await traces.waitInit()
const context = otelCarrier
? traces.getContextFromCarrier(otelCarrier)
@ -208,7 +349,7 @@ export function createFetchModuleFunction(
context
? { context }
: {},
span => fetcher(span, url, importer, environment, cacheFs, options),
span => fetcher.fetch(span, url, importer, environment, cacheFs, options),
)
}
}
@ -218,9 +359,7 @@ SOURCEMAPPING_URL += 'ppingURL'
const MODULE_RUNNER_SOURCEMAPPING_SOURCE = '//# sourceMappingSource=vite-generated'
function processResultSource(environment: DevEnvironment, result: FetchResult): FetchResult & {
transformResult?: TransformResult | null
} {
function processResultSource(environment: DevEnvironment, result: FetchResult): FetchResult {
if (!('code' in result)) {
return result
}
@ -235,7 +374,6 @@ function processResultSource(environment: DevEnvironment, result: FetchResult):
return {
...result,
code: node?.transformResult?.code || result.code,
transformResult: node?.transformResult,
}
}
@ -299,6 +437,32 @@ function getCachedResult(result: Extract<FetchResult, { code: string }>, tmp: st
}
}
const MODULE_RUNNER_SOURCEMAPPING_REGEXP = new RegExp(
`//# ${SOURCEMAPPING_URL}=data:application/json;base64,(.+)`,
)
function extractSourceMap(code: string): null | Rollup.SourceMap {
const pattern = `//# ${SOURCEMAPPING_URL}=data:application/json;base64,`
const lastIndex = code.lastIndexOf(pattern)
if (lastIndex === -1) {
return null
}
const mapString = MODULE_RUNNER_SOURCEMAPPING_REGEXP.exec(
code.slice(lastIndex),
)?.[1]
if (!mapString) {
return null
}
const sourceMap = JSON.parse(Buffer.from(mapString, 'base64').toString('utf-8'))
// remove source map mapping added by "inlineSourceMap" to keep the original behaviour of transformRequest
if (sourceMap.mappings.startsWith('AAAA,CAAA;')) {
// 9 because we want to only remove "AAAA,CAAA", but keep ; at the start
sourceMap.mappings = sourceMap.mappings.slice(9)
}
return sourceMap
}
// serialize rollup error on server to preserve details as a test error
export function handleRollupError(e: unknown): never {
if (
@ -321,35 +485,3 @@ export function handleRollupError(e: unknown): never {
}
throw e
}
/**
* Performs an atomic write operation using the write-then-rename pattern.
*
* Why we need this:
* - Ensures file integrity by never leaving partially written files on disk
* - Prevents other processes from reading incomplete data during writes
* - Particularly important for test files where incomplete writes could cause test failures
*
* The implementation writes to a temporary file first, then renames it to the target path.
* This rename operation is atomic on most filesystems (including POSIX-compliant ones),
* guaranteeing that other processes will only ever see the complete file.
*
* Added in https://github.com/vitest-dev/vitest/pull/7531
*/
async function atomicWriteFile(realFilePath: string, data: string): Promise<void> {
const dir = dirname(realFilePath)
const tmpFilePath = join(dir, `.tmp-${Date.now()}-${Math.random().toString(36).slice(2)}`)
try {
await writeFile(tmpFilePath, data, 'utf-8')
await rename(tmpFilePath, realFilePath)
}
finally {
try {
if (await stat(tmpFilePath)) {
await unlink(tmpFilePath)
}
}
catch {}
}
}

View File

@ -1,6 +1,7 @@
import type { DevEnvironment } from 'vite'
import type { ResolvedConfig } from '../types/config'
import type { VitestFetchFunction } from './fetchModule'
import { readFile } from 'node:fs/promises'
import { VitestModuleEvaluator } from '#module-evaluator'
import { ModuleRunner } from 'vite/module-runner'
import { normalizeResolvedIdToUrl } from './normalizeUrl'
@ -28,6 +29,10 @@ export class ServerModuleRunner extends ModuleRunner {
}
try {
const result = await fetcher(data[0], data[1], environment, false, data[2])
if ('tmp' in result) {
const code = await readFile(result.tmp)
return { result: { ...result, code } }
}
return { result }
}
catch (error) {

View File

@ -1,6 +1,6 @@
import type { UserConfig as ViteConfig, Plugin as VitePlugin } from 'vite'
import type { TestProject } from '../project'
import type { BrowserConfigOptions, ResolvedConfig, TestProjectInlineConfiguration } from '../types/config'
import type { BrowserConfigOptions, ResolvedConfig, TestProjectInlineConfiguration, UserConfig } from '../types/config'
import { existsSync, readFileSync } from 'node:fs'
import { deepMerge } from '@vitest/utils/helpers'
import { basename, dirname, relative, resolve } from 'pathe'
@ -93,6 +93,16 @@ export function WorkspaceVitestPlugin(
}
}
const vitestConfig: UserConfig = {
name: { label: name, color },
}
// always inherit the global `fsModuleCache` value even without `extends: true`
if (testConfig.experimental?.fsModuleCache == null && project.vitest.config.experimental?.fsModuleCache !== null) {
vitestConfig.experimental ??= {}
vitestConfig.experimental.fsModuleCache = project.vitest.config.experimental.fsModuleCache
}
return {
base: '/',
environments: {
@ -100,9 +110,7 @@ export function WorkspaceVitestPlugin(
dev: {},
},
},
test: {
name: { label: name, color },
},
test: vitestConfig,
}
},
},

View File

@ -561,12 +561,10 @@ export class TestProject {
this._serializedDefines = createDefinesScript(server.config.define)
this._fetcher = createFetchModuleFunction(
this._resolver,
this._config,
this.vitest._fsCache,
this.vitest._traces,
this.tmpDir,
{
dumpFolder: this.config.dumpDir,
readFromDump: this.config.server.debug?.load ?? process.env.VITEST_DEBUG_LOAD_DUMP != null,
},
)
const environment = server.environments.__vitest__

View File

@ -9,12 +9,21 @@ import { dirname, extname, join, resolve } from 'pathe'
import { isWindows } from '../utils/env'
export class VitestResolver {
private options: ExternalizeOptions
public readonly options: ExternalizeOptions
private externalizeCache = new Map<string, Promise<string | false>>()
constructor(cacheDir: string, config: ResolvedConfig) {
// sorting to make cache consistent
const inline = config.server.deps?.inline
if (Array.isArray(inline)) {
inline.sort()
}
const external = config.server.deps?.external
if (Array.isArray(external)) {
external.sort()
}
this.options = {
moduleDirectories: config.deps.moduleDirectories,
moduleDirectories: config.deps.moduleDirectories?.sort(),
inlineFiles: config.setupFiles.flatMap((file) => {
if (file.startsWith('file://')) {
return file
@ -23,8 +32,8 @@ export class VitestResolver {
return [resolvedId, pathToFileURL(resolvedId).href]
}),
cacheDir,
inline: config.server.deps?.inline,
external: config.server.deps?.external,
inline,
external,
}
}

View File

@ -826,8 +826,21 @@ export interface InlineConfig {
*/
attachmentsDir?: string
/** @experimental */
/**
* Experimental features
*
* @experimental
*/
experimental?: {
/**
* Enable caching of modules on the file system between reruns.
*/
fsModuleCache?: boolean
/**
* Path relative to the root of the project where the fs module cache will be stored.
* @default node_modules/.experimental-vitest-cache
*/
fsModuleCachePath?: string
/**
* {@link https://vitest.dev/guide/open-telemetry}
*/
@ -965,6 +978,12 @@ export interface UserConfig extends InlineConfig {
* @default '.vitest-reports'
*/
mergeReports?: string
/**
* Delete all Vitest caches, including `experimental.fsModuleCache`.
* @experimental
*/
clearCache?: boolean
}
export type OnUnhandledErrorCallback = (error: (TestError | Error) & { type: string }) => boolean | void

View File

@ -53,6 +53,13 @@ export interface CoverageProvider {
pluginCtx: any,
) => TransformResult | Promise<TransformResult>
/**
* Return `true` if this file is transformed by the coverage provider.
* This is used to generate the persistent file hash by `fsModuleCache`
* @experimental
*/
requiresTransform?: (id: string) => boolean
/** Callback that's called when the coverage is enabled via a programmatic `enableCoverage` API. */
onEnabled?: () => void | Promise<void>
}

View File

@ -1,3 +1,4 @@
import type { CacheKeyIdGenerator } from '../cache/fsModuleCache'
import type { Vitest } from '../core'
import type { TestProject } from '../project'
import type { TestProjectConfiguration } from './config'
@ -6,4 +7,14 @@ export interface VitestPluginContext {
vitest: Vitest
project: TestProject
injectTestProjects: (config: TestProjectConfiguration | TestProjectConfiguration[]) => Promise<TestProject[]>
/**
* Define a generator that will be applied before hashing the cache key.
*
* Use this to make sure Vitest generates correct hash. It is a good idea
* to define this function if your plugin can be registered with different options.
*
* This is called only if `experimental.fsModuleCache` is defined.
* @experimental
*/
experimental_defineCacheKeyGenerator: (callback: CacheKeyIdGenerator) => void
}

View File

@ -5,6 +5,7 @@ export const version: string = Vitest.version
export { isValidApiRequest } from '../api/check'
export { escapeTestName } from '../node/ast-collect'
export type { CacheKeyIdGenerator, CacheKeyIdGeneratorContext } from '../node/cache/fsModuleCache'
export { parseCLI } from '../node/cli/cac'
export type { CliParseOptions } from '../node/cli/cac'
export type { CliOptions } from '../node/cli/cli-api'

View File

@ -117,6 +117,9 @@ export interface SerializedConfig {
includeSamples: boolean
} | undefined
serializedDefines: string
experimental: {
fsModuleCache: boolean
}
}
export interface SerializedCoverageConfig {

6
pnpm-lock.yaml generated
View File

@ -1167,6 +1167,12 @@ importers:
specifier: 'catalog:'
version: 8.18.3
test/cache:
devDependencies:
vitest:
specifier: workspace:*
version: link:../../packages/vitest
test/cli:
devDependencies:
'@opentelemetry/sdk-node':

View File

@ -0,0 +1,5 @@
import { test, expect } from 'vitest'
test('replaced variable is the same', () => {
expect(__REPLACED__).toBe(process.env.REPLACED)
})

View File

@ -0,0 +1,17 @@
import { defineConfig } from 'vitest/config'
export default defineConfig({
plugins: [
{
name: 'test:replacer',
transform(code) {
return code.replace('__REPLACED__', JSON.stringify(process.env.REPLACED))
},
configureVitest(ctx) {
ctx.experimental_defineCacheKeyGenerator(() => {
return false
})
},
},
],
})

View File

@ -0,0 +1,12 @@
import { defineConfig } from 'vitest/config'
export default defineConfig({
plugins: [
{
name: 'test:replacer',
transform(code) {
return code.replace('__REPLACED__', JSON.stringify(process.env.REPLACED))
},
},
],
})

View File

@ -0,0 +1,17 @@
import { defineConfig } from 'vitest/config'
export default defineConfig({
plugins: [
{
name: 'test:replacer',
transform(code) {
return code.replace('__REPLACED__', JSON.stringify(process.env.REPLACED))
},
configureVitest(ctx) {
ctx.experimental_defineCacheKeyGenerator(() => {
return String(process.env.REPLACED)
})
},
},
],
})

View File

@ -0,0 +1,6 @@
import { test, expect, inject } from 'vitest'
test('replaced variable is the same', () => {
const files = import.meta.glob('./generated/*')
expect(Object.keys(files)).toEqual(inject('generated'))
})

View File

@ -0,0 +1,3 @@
import { defineConfig } from 'vitest/config'
export default defineConfig({})

11
test/cache/package.json vendored Normal file
View File

@ -0,0 +1,11 @@
{
"name": "@vitest/test-cache",
"type": "module",
"private": true,
"scripts": {
"test": "vitest"
},
"devDependencies": {
"vitest": "workspace:*"
}
}

View File

@ -0,0 +1,148 @@
import { expect, test } from 'vitest'
import { runVitest } from '../../test-utils'
test('if no cache key generator is defined, the hash is invalid', async () => {
process.env.REPLACED = 'value1'
const { errorTree: errorTree1 } = await runVitest({
root: './fixtures/dynamic-cache-key',
config: './vitest.config.fails.js',
experimental: {
fsModuleCache: true,
fsModuleCachePath: './node_modules/.vitest-fs-cache',
},
reporters: [
{
async onInit(vitest) {
// make sure cache is empty
await vitest.experimental_clearCache()
},
},
],
})
expect(errorTree1()).toMatchInlineSnapshot(`
{
"replaced.test.js": {
"replaced variable is the same": "passed",
},
}
`)
process.env.REPLACED = 'value2'
const { errorTree: errorTree2 } = await runVitest({
root: './fixtures/dynamic-cache-key',
config: './vitest.config.fails.js',
experimental: {
fsModuleCache: true,
fsModuleCachePath: './node_modules/.vitest-fs-cache',
},
})
expect(errorTree2()).toMatchInlineSnapshot(`
{
"replaced.test.js": {
"replaced variable is the same": [
"expected 'value1' to be 'value2' // Object.is equality",
],
},
}
`)
})
test('if cache key generator is defined, the hash is valid', async () => {
process.env.REPLACED = 'value1'
const { errorTree: errorTree1 } = await runVitest({
root: './fixtures/dynamic-cache-key',
config: './vitest.config.passes.js',
experimental: {
fsModuleCache: true,
fsModuleCachePath: './node_modules/.vitest-fs-cache',
},
reporters: [
{
async onInit(vitest) {
// make sure cache is empty
await vitest.experimental_clearCache()
},
},
],
})
expect(errorTree1()).toMatchInlineSnapshot(`
{
"replaced.test.js": {
"replaced variable is the same": "passed",
},
}
`)
process.env.REPLACED = 'value2'
const { errorTree: errorTree2 } = await runVitest({
root: './fixtures/dynamic-cache-key',
config: './vitest.config.passes.js',
experimental: {
fsModuleCache: true,
fsModuleCachePath: './node_modules/.vitest-fs-cache',
},
})
expect(errorTree2()).toMatchInlineSnapshot(`
{
"replaced.test.js": {
"replaced variable is the same": "passed",
},
}
`)
})
test('if cache key generator bails out, the file is not cached', async () => {
process.env.REPLACED = 'value1'
const { errorTree: errorTree1 } = await runVitest({
root: './fixtures/dynamic-cache-key',
config: './vitest.config.bails.js',
experimental: {
fsModuleCache: true,
fsModuleCachePath: './node_modules/.vitest-fs-cache',
},
reporters: [
{
async onInit(vitest) {
// make sure cache is empty
await vitest.experimental_clearCache()
},
},
],
})
expect(errorTree1()).toMatchInlineSnapshot(`
{
"replaced.test.js": {
"replaced variable is the same": "passed",
},
}
`)
process.env.REPLACED = 'value2'
const { errorTree: errorTree2 } = await runVitest({
root: './fixtures/dynamic-cache-key',
config: './vitest.config.bails.js',
experimental: {
fsModuleCache: true,
fsModuleCachePath: './node_modules/.vitest-fs-cache',
},
})
expect(errorTree2()).toMatchInlineSnapshot(`
{
"replaced.test.js": {
"replaced variable is the same": "passed",
},
}
`)
})

59
test/cache/test/importMetaGlob.test.ts vendored Normal file
View File

@ -0,0 +1,59 @@
import { expect, test } from 'vitest'
import { runVitest, useFS } from '../../test-utils'
test('if file has import.meta.glob, it\'s not cached', async () => {
const { createFile } = useFS('./fixtures/import-meta-glob/generated', {
1: '1',
2: '2',
}, false)
const { errorTree: errorTree1 } = await runVitest({
root: './fixtures/import-meta-glob',
provide: {
generated: ['./generated/1', './generated/2'],
},
experimental: {
fsModuleCache: true,
fsModuleCachePath: './node_modules/.vitest-fs-cache',
},
})
expect(errorTree1()).toMatchInlineSnapshot(`
{
"glob.test.js": {
"replaced variable is the same": "passed",
},
}
`)
createFile('3', '3')
const { errorTree: errorTree2 } = await runVitest({
root: './fixtures/import-meta-glob',
provide: {
generated: [
'./generated/1',
'./generated/2',
'./generated/3',
],
},
experimental: {
fsModuleCache: true,
fsModuleCachePath: './node_modules/.vitest-fs-cache',
},
})
expect(errorTree2()).toMatchInlineSnapshot(`
{
"glob.test.js": {
"replaced variable is the same": "passed",
},
}
`)
})
declare module 'vitest' {
export interface ProvidedContext {
generated: string[]
}
}

14
test/cache/vitest.config.ts vendored Normal file
View File

@ -0,0 +1,14 @@
import { defineConfig } from 'vite'
export default defineConfig({
test: {
include: ['test/**.test.ts'],
includeTaskLocation: true,
reporters: ['verbose'],
testTimeout: 60_000,
fileParallelism: false,
chaiConfig: {
truncateThreshold: 999,
},
},
})

View File

@ -1,7 +1,9 @@
import { resolve } from 'pathe'
import { expect, test } from 'vitest'
import { expect, it, test } from 'vitest'
import { createVitest } from 'vitest/node'
import { runVitest } from '../../test-utils'
test('can pass down the config as a module', async () => {
const vitest = await createVitest('test', {
config: '@test/test-dep-config',
@ -11,3 +13,39 @@ test('can pass down the config as a module', async () => {
resolve(import.meta.dirname, '../deps/test-dep-config/index.js'),
)
})
it('correctly inherit from the cli', async () => {
const { ctx } = await runVitest({
root: 'fixtures/workspace-flags',
logHeapUsage: true,
allowOnly: true,
sequence: {
seed: 123,
},
testTimeout: 5321,
pool: 'forks',
globals: true,
expandSnapshotDiff: true,
retry: 6,
testNamePattern: 'math',
passWithNoTests: true,
bail: 100,
})
const project = ctx!.projects[0]
const config = project.config
expect(config).toMatchObject({
logHeapUsage: true,
allowOnly: true,
sequence: expect.objectContaining({
seed: 123,
}),
testTimeout: 5321,
pool: 'forks',
globals: true,
expandSnapshotDiff: true,
retry: 6,
passWithNoTests: true,
bail: 100,
})
expect(config.testNamePattern?.test('math')).toBe(true)
})

View File

@ -1,38 +0,0 @@
import { expect, it } from 'vitest'
import { runVitest } from '../../test-utils'
it('correctly inherit from the cli', async () => {
const { ctx } = await runVitest({
root: 'fixtures/workspace-flags',
logHeapUsage: true,
allowOnly: true,
sequence: {
seed: 123,
},
testTimeout: 5321,
pool: 'forks',
globals: true,
expandSnapshotDiff: true,
retry: 6,
testNamePattern: 'math',
passWithNoTests: true,
bail: 100,
})
const project = ctx!.projects[0]
const config = project.config
expect(config).toMatchObject({
logHeapUsage: true,
allowOnly: true,
sequence: expect.objectContaining({
seed: 123,
}),
testTimeout: 5321,
pool: 'forks',
globals: true,
expandSnapshotDiff: true,
retry: 6,
passWithNoTests: true,
bail: 100,
})
expect(config.testNamePattern?.test('math')).toBe(true)
})

View File

@ -89,3 +89,40 @@ describe.each([
}).rejects.toThrowError(`Inspector host cannot be a URL. Use "host:port" instead of "${url}"`)
})
})
it('experimental fsModuleCache is inherited in a project', async () => {
const v = await vitest({}, {
experimental: {
fsModuleCache: true,
},
projects: [
{
test: {
name: 'project',
},
},
],
})
expect(v.config.experimental.fsModuleCache).toBe(true)
expect(v.projects[0].config.experimental.fsModuleCache).toBe(true)
})
it('project overrides experimental fsModuleCache', async () => {
const v = await vitest({}, {
experimental: {
fsModuleCache: true,
},
projects: [
{
test: {
name: 'project',
experimental: {
fsModuleCache: false,
},
},
},
],
})
expect(v.config.experimental.fsModuleCache).toBe(true)
expect(v.projects[0].config.experimental.fsModuleCache).toBe(false)
})

View File

@ -1,8 +1,16 @@
import type { Options } from 'tinyexec'
import type { UserConfig as ViteUserConfig } from 'vite'
import type { WorkerGlobalState } from 'vitest'
import type { SerializedConfig, WorkerGlobalState } from 'vitest'
import type { TestProjectConfiguration } from 'vitest/config'
import type { TestCollection, TestModule, TestSpecification, TestUserConfig, Vitest, VitestRunMode } from 'vitest/node'
import type {
TestCollection,
TestModule,
TestResult,
TestSpecification,
TestUserConfig,
Vitest,
VitestRunMode,
} from 'vitest/node'
import { webcrypto as crypto } from 'node:crypto'
import fs from 'node:fs'
import { Readable, Writable } from 'node:stream'
@ -70,6 +78,8 @@ export async function runVitest(
stdin.isTTY = true
stdin.setRawMode = () => stdin
const cli = new Cli({ stdin, stdout, stderr, preserveAnsi: runnerOptions.preserveAnsi })
// @ts-expect-error not typed global
const currentConfig: SerializedConfig = __vitest_worker__.ctx.config
let ctx: Vitest | undefined
let thrown = false
@ -88,6 +98,11 @@ export async function runVitest(
NO_COLOR: 'true',
...rest.env,
},
// override cache config with the one that was used to run `vitest` formt the CLI
experimental: {
fsModuleCache: currentConfig.experimental.fsModuleCache,
...rest.experimental,
},
}, {
...viteOverrides,
server: {
@ -141,6 +156,17 @@ export async function runVitest(
vitest: cli,
stdout: cli.stdout,
stderr: cli.stderr,
get results() {
return ctx?.state.getTestModules() || []
},
errorTree() {
return buildTestTree(ctx?.state.getTestModules() || [], (result) => {
if (result.state === 'failed') {
return result.errors.map(e => e.message)
}
return result.state
})
},
testTree() {
return buildTestTree(ctx?.state.getTestModules() || [])
},
@ -306,10 +332,10 @@ export default config
return `export default ${JSON.stringify(content)}`
}
export function useFS<T extends TestFsStructure>(root: string, structure: T) {
export function useFS<T extends TestFsStructure>(root: string, structure: T, ensureConfig = true) {
const files = new Set<string>()
const hasConfig = Object.keys(structure).some(file => file.includes('.config.'))
if (!hasConfig) {
if (ensureConfig && !hasConfig) {
;(structure as any)['./vitest.config.js'] = {}
}
for (const file in structure) {
@ -338,9 +364,10 @@ export function useFS<T extends TestFsStructure>(root: string, structure: T) {
throw new Error(`file ${file} is outside of the test file system`)
}
const filepath = resolve(root, file)
if (!files.has(filepath)) {
if (files.has(filepath)) {
throw new Error(`file ${file} already exists in the test file system`)
}
files.add(filepath)
createFile(filepath, content)
},
statFile: (file: string): fs.Stats => {
@ -392,7 +419,7 @@ export class StableTestFileOrderSorter {
}
}
function buildTestTree(testModules: TestModule[]) {
function buildTestTree(testModules: TestModule[], onResult?: (result: TestResult) => unknown) {
type TestTree = Record<string, any>
function walkCollection(collection: TestCollection): TestTree {
@ -406,7 +433,12 @@ function buildTestTree(testModules: TestModule[]) {
}
else if (child.type === 'test') {
const result = child.result()
node[child.name] = result.state
if (onResult) {
node[child.name] = onResult(result)
}
else {
node[child.name] = result.state
}
}
}