import type { Vitest } from 'vitest/node'
import type { JsonTestResults } from 'vitest/reporters'
import { readdirSync } from 'node:fs'
import { readFile } from 'node:fs/promises'
import { beforeAll, describe, expect, onTestFailed, test } from 'vitest'
import { rolldownVersion } from 'vitest/node'
import { instances, provider, runBrowserTests } from './utils'
function noop() {}
describe('running browser tests', async () => {
let stderr: string
let stdout: string
let browserResultJson: JsonTestResults
let passedTests: any[]
let failedTests: any[]
let vitest: Vitest
const events: string[] = []
beforeAll(async () => {
({
stderr,
stdout,
ctx: vitest,
} = await runBrowserTests({
allowOnly: true,
reporters: [
{
onBrowserInit(project) {
events.push(`onBrowserInit ${project.name}`)
},
},
'json',
{
onInit: noop,
onTestRunStart: noop,
onTestModuleCollected: noop,
onTestRunEnd: noop,
onTestRemoved: noop,
onWatcherStart: noop,
onWatcherRerun: noop,
onServerRestart: noop,
onUserConsoleLog: noop,
},
'default',
],
}))
const browserResult = await readFile('./browser.json', 'utf-8')
browserResultJson = JSON.parse(browserResult)
const getPassed = results => results.filter(result => result.status === 'passed' && !result.message)
const getFailed = results => results.filter(result => result.status === 'failed')
passedTests = getPassed(browserResultJson.testResults)
failedTests = getFailed(browserResultJson.testResults)
})
test('tests are actually running', () => {
expect(stderr).toBe('')
const testFiles = browserResultJson.testResults.map(t => t.name)
vitest.projects.forEach((project) => {
// the order is non-deterministic
expect(events).toContain(`onBrowserInit ${project.name}`)
})
// test files are optimized automatically
expect(vitest.projects.map(p => p.browser?.vite.config.optimizeDeps.entries))
.toEqual(vitest.projects.map(() => expect.arrayContaining(testFiles)))
const testFilesCount = readdirSync('./test')
.filter(n => n.includes('.test.'))
.length + 1 // 1 is in-source-test
expect(browserResultJson.testResults).toHaveLength(testFilesCount * instances.length)
expect(passedTests).toHaveLength(browserResultJson.testResults.length)
expect(failedTests).toHaveLength(0)
})
test('runs in-source tests', () => {
expect(stdout).toContain('src/actions.ts')
const actionsTest = passedTests.find(t => t.name.includes('/actions.ts'))
expect(actionsTest).toBeDefined()
expect(actionsTest.assertionResults).toHaveLength(1)
})
test('unsubscribes cancel listeners after run', async () => {
// should not throw birpc closing errors
await expect(vitest.cancelCurrentRun('keyboard-input')).resolves.not.toThrow()
})
})
describe('console logging tests', async () => {
let stderr: string
let stdout: string
beforeAll(async () => {
({
stderr,
stdout,
} = await runBrowserTests({
root: './fixtures/print-logs',
}))
})
test('logs are redirected to stdout', () => {
expect(stdout).toContain('stdout | test/logs.test.ts > logging to stdout')
expect(stdout).toContain('hello from console.log')
expect(stdout).toContain('hello from console.info')
expect(stdout).toContain('hello from console.debug')
expect(stdout).toContain(`
{
"hello": "from dir",
}
`.trim())
expect(stdout).toContain(`
{
"hello": "from dirxml",
}
`.trim())
expect(stdout).toContain('dom
')
expect(stdout).toContain('default: 1')
expect(stdout).toContain('default: 2')
expect(stdout).toContain('default: 3')
expect(stdout).toContain('count: 1')
expect(stdout).toContain('count: 2')
expect(stdout).toContain('count: 3')
expect(stdout).toMatch(/default: [\d.]+ ms/)
expect(stdout).toMatch(/time: [\d.]+ ms/)
expect(stdout).toMatch(/\[console-time-fake\]: [\d.]+ ms/)
expect(stdout).not.toContain('[console-time-fake]: 0 ms')
})
test('logs are redirected to stderr', () => {
expect(stderr).toContain('stderr | test/logs.test.ts > logging to stderr')
expect(stderr).toContain('hello from console.error')
expect(stderr).toContain('hello from console.warn')
expect(stderr).toContain('Timer "invalid timeLog" does not exist')
expect(stderr).toContain('Timer "invalid timeEnd" does not exist')
// safari logs the stack files with @https://...
expect(stderr).toMatch(/hello from console.trace\s+(\w+|@)/)
})
test(`logs have stack traces`, () => {
expect(stdout).toMatch(`
log with a stack
❯ test/logs.test.ts:58:10
`.trim())
expect(stderr).toMatch(`
error with a stack
❯ test/logs.test.ts:59:10
`.trim())
// console.trace processes the stack trace correctly
expect(stderr).toMatch('test/logs.test.ts:60:10')
if (instances.some(({ browser }) => browser === 'webkit')) {
// safari print stack trace in a different place
expect(stdout).toMatch(`
log with a stack
❯ test/logs.test.ts:58:14
`.trim())
expect(stderr).toMatch(`
error with a stack
❯ test/logs.test.ts:59:16
`.trim())
// console.trace processes the stack trace correctly
expect(stderr).toMatch('test/logs.test.ts:60:16')
}
})
test('popup apis should log a warning', () => {
expect(stderr).toContain('Vitest encountered a `alert("test")`')
expect(stderr).toContain('Vitest encountered a `confirm("test")`')
expect(stderr).toContain('Vitest encountered a `prompt("test")`')
})
})
test(`stack trace points to correct file in every browser when failed`, async () => {
expect.assertions(15)
const { stderr } = await runBrowserTests({
root: './fixtures/failing',
reporters: [
'default',
{
onTestCaseReady(testCase) {
if (testCase.name !== 'correctly fails and prints a diff') {
return
}
if (testCase.project.name === 'chromium' || testCase.project.name === 'chrome') {
expect(testCase.result().errors[0].stacks).toEqual([
{
line: 11,
column: 12,
file: testCase.module.moduleId,
method: '',
},
])
}
},
},
],
})
expect(stderr).toContain('expected 1 to be 2')
expect(stderr).toMatch(/- 2\s+\+ 1/)
// expect(stderr).toContain('Expected to be')
// expect(stderr).toContain('But got')
expect(stderr).toContain('Failure screenshot')
expect(stderr).toContain('__screenshots__/failing')
expect(stderr).toContain('Access denied to "/inaccesible/path".')
// depending on the browser it references either `.toBe()` or `expect()`
expect(stderr).toMatch(/failing.test.ts:11:(12|17)/)
// column is 18 in safari, 8 in others
expect(stderr).toMatch(/throwError src\/error.ts:8:(18|8)/)
expect(stderr).toContain('The call was not awaited. This method is asynchronous and must be awaited; otherwise, the call will not start to avoid unhandled rejections.')
expect(stderr).toMatch(/failing.test.ts:19:(27|36)/)
expect(stderr).toMatch(/failing.test.ts:20:(27|33)/)
expect(stderr).toMatch(/failing.test.ts:21:(27|39)/)
expect(stderr).toMatch(/bundled-lib\/src\/b.js:2:(9|19)/)
expect(stderr).toMatch(/bundled-lib\/src\/index.js:5:(16|18)/)
// index() is called from a bundled file
expect(stderr).toMatch(/failing.test.ts:25:(2|8)/)
})
test('user-event', async () => {
const { stdout, stderr } = await runBrowserTests({
root: './fixtures/user-event',
})
if (provider.name !== 'webdriverio') {
expect(stderr).toBe('')
}
onTestFailed(() => console.error(stderr))
instances.forEach(({ browser }) => {
expect(stdout).toReportPassedTest('cleanup-retry.test.ts', browser)
expect(stdout).toReportPassedTest('cleanup1.test.ts', browser)
expect(stdout).toReportPassedTest('cleanup2.test.ts', browser)
expect(stdout).toReportPassedTest('keyboard.test.ts', browser)
expect(stdout).toReportPassedTest('clipboard.test.ts', browser)
})
})
test('timeout settings', async () => {
const { stderr } = await runBrowserTests({
root: './fixtures/timeout',
})
expect(stderr).toContain('Matcher did not succeed in time.')
if (provider.name === 'playwright') {
expect(stderr).toContain('locator.click: Timeout 500ms exceeded.')
expect(stderr).toContain('locator.click: Timeout 345ms exceeded.')
}
if (provider.name === 'webdriverio') {
expect(stderr).toContain('Cannot find element with locator')
}
})
test('viewport', async () => {
const { stdout, stderr } = await runBrowserTests({
root: './fixtures/viewport',
})
expect(stderr).toBe('')
instances.forEach(({ browser }) => {
expect(stdout).toReportPassedTest('basic.test.ts', browser)
})
})
test('in-source tests don\'t run when the module is imported by the test', async () => {
const { stderr, stdout } = await runBrowserTests({}, ['mocking.test.ts'])
expect(stderr).toBe('')
instances.forEach(({ browser }) => {
expect(stdout).toReportPassedTest('test/mocking.test.ts', browser)
})
// there is only one file with one test inside
// if this stops working, it will report twice as much tests
expect(stdout).toContain(`Test Files ${instances.length} passed`)
expect(stdout).toContain(`Tests ${instances.length} passed`)
})
test('in-source tests run correctly when filtered', async () => {
const { stderr, stdout } = await runBrowserTests({}, ['actions.ts'])
expect(stderr).toBe('')
instances.forEach(({ browser }) => {
expect(stdout).toReportPassedTest('src/actions.ts', browser)
})
// there is only one file with one test inside
expect(stdout).toContain(`Test Files ${instances.length} passed`)
expect(stdout).toContain(`Tests ${instances.length} passed`)
})
test('re-evaluate setupFiles on each test run even when isolate is false', async () => {
const { exitCode, stderr, stdout } = await runBrowserTests({
root: './fixtures/isolate-and-setup-file',
})
expect(stderr).toBe('')
expect(exitCode).toBe(0)
instances.forEach(({ browser }) => {
expect(stdout).toReportPassedTest('a.test.ts', browser)
expect(stdout).toReportPassedTest('b.test.ts', browser)
})
})
test.runIf(provider.name === 'playwright')('timeout hooks', async ({ onTestFailed }) => {
const { stderr } = await runBrowserTests({
root: './fixtures/timeout-hooks',
})
onTestFailed(() => {
console.error(stderr)
})
const lines = stderr.split('\n')
const timeoutErrorsIndexes = []
lines.forEach((line, index) => {
if (line.includes('TimeoutError:')) {
timeoutErrorsIndexes.push(index)
}
})
const snapshot = timeoutErrorsIndexes.map((index) => {
return [
lines[index - 1],
lines[index].replace(/Timeout \d+ms exceeded/, 'Timeout exceeded'),
lines[index + 4],
].join('\n')
}).sort().join('\n\n')
// rolldown has better source maps
if (rolldownVersion) {
expect(snapshot).toMatchInlineSnapshot(`
" FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:44:45
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:26:45
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:35:45
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:17:45
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:7:33
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:70:47
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:79:47
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:54:47
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:61:47
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:44:45
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:26:45
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:35:45
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:17:45
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:7:33
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:70:47
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:79:47
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:54:47
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:61:47
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:44:45
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:26:45
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:35:45
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:17:45
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:7:33
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:70:47
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:79:47
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:54:47
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:61:47"
`)
}
else {
expect(snapshot).toMatchInlineSnapshot(`
" FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:39:45
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:23:45
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:31:45
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:15:45
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:6:33
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:62:47
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:70:47
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:48:47
FAIL |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:54:47
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:39:45
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:23:45
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:31:45
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:15:45
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:6:33
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:62:47
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:70:47
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:48:47
FAIL |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:54:47
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:39:51
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:23:51
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:31:51
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:15:51
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:6:39
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:62:53
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:70:53
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:48:53
FAIL |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
TimeoutError: locator.click: Timeout exceeded.
❯ hooks-timeout.test.ts:54:53"
`)
}
// page.getByRole('code').click()
expect(stderr).toContain('locator.click: Timeout')
// playwright error is proxied from the server to the client and back correctly
expect(stderr).toContain('waiting for locator(\'[data-vitest="true"]\').contentFrame().getByRole(\'code\')')
expect(stderr).toMatch(/hooks-timeout.test.ts:6:(33|39)/)
// await expect.element().toBeVisible()
expect(stderr).toContain('Cannot find element with locator: getByRole(\'code\')')
expect(stderr).toMatch(/hooks-timeout.test.ts:10:(49|61)/)
}, 120_000 * 3)