From 16d13d98115a1a3b00b2b95e8829cc6c8d00ebe3 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Wed, 18 Feb 2026 09:43:08 +0900 Subject: [PATCH] fix: hooks should respect `maxConcurrency` (#9653) --- docs/config/maxconcurrency.md | 4 +- docs/config/sequence.md | 2 +- docs/guide/cli-generated.md | 2 +- docs/guide/parallelism.md | 2 + packages/runner/src/run.ts | 45 +- .../runner/src/utils/limit-concurrency.ts | 64 +- packages/vitest/src/node/cli/cli-config.ts | 2 +- .../fails/concurrent-suite-deadlock.test.ts | 44 - .../fails/concurrent-test-deadlock.test.ts | 40 - .../cli/test/__snapshots__/fails.test.ts.snap | 10 - test/cli/test/around-each.test.ts | 96 +- test/cli/test/concurrent.test.ts | 1076 +++++++++++++++++ test/cli/test/expect-task.test.ts | 12 +- test/core/test/concurrent-suite.test.ts | 45 +- test/test-utils/index.ts | 45 +- 15 files changed, 1338 insertions(+), 151 deletions(-) delete mode 100644 test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts delete mode 100644 test/cli/fixtures/fails/concurrent-test-deadlock.test.ts create mode 100644 test/cli/test/concurrent.test.ts diff --git a/docs/config/maxconcurrency.md b/docs/config/maxconcurrency.md index 6026efe77c05..deb88476b2a0 100644 --- a/docs/config/maxconcurrency.md +++ b/docs/config/maxconcurrency.md @@ -9,6 +9,6 @@ outline: deep - **Default**: `5` - **CLI**: `--max-concurrency=10`, `--maxConcurrency=10` -A number of tests that are allowed to run at the same time marked with `test.concurrent`. +The maximum number of tests and hooks that can run at the same time when using `test.concurrent` or `describe.concurrent`. -Test above this limit will be queued to run when available slot appears. +The hook execution order within a single group is also controlled by [`sequence.hooks`](/config/sequence#sequence-hooks). With `sequence.hooks: 'parallel'`, the execution is bounded by the same limit of [`maxConcurrency`](/config/maxconcurrency). diff --git a/docs/config/sequence.md b/docs/config/sequence.md index f3b05eab398f..9fc65fb77dbf 100644 --- a/docs/config/sequence.md +++ b/docs/config/sequence.md @@ -145,7 +145,7 @@ Changes the order in which hooks are executed. - `stack` will order "after" hooks in reverse order, "before" hooks will run in the order they were defined - `list` will order all hooks in the order they are defined -- `parallel` will run hooks in a single group in parallel (hooks in parent suites will still run before the current suite's hooks) +- `parallel` runs hooks in a single group in parallel (hooks in parent suites still run before the current suite's hooks). The actual number of simultaneously running hooks is limited by [`maxConcurrency`](/config/maxconcurrency). ::: tip This option doesn't affect [`onTestFinished`](/api/hooks#ontestfinished). It is always called in reverse order. diff --git a/docs/guide/cli-generated.md b/docs/guide/cli-generated.md index 289348a98aac..52f67fc9a247 100644 --- a/docs/guide/cli-generated.md +++ b/docs/guide/cli-generated.md @@ -781,7 +781,7 @@ Default timeout of a teardown function in milliseconds (default: `10000`) - **CLI:** `--maxConcurrency ` - **Config:** [maxConcurrency](/config/maxconcurrency) -Maximum number of concurrent tests in a suite (default: `5`) +Maximum number of concurrent tests and suites during test file execution (default: `5`) ### expect.requireAssertions diff --git a/docs/guide/parallelism.md b/docs/guide/parallelism.md index c5772b20865d..c2033531e1ff 100644 --- a/docs/guide/parallelism.md +++ b/docs/guide/parallelism.md @@ -22,6 +22,8 @@ Unlike _test files_, Vitest runs _tests_ in sequence. This means that tests insi Vitest supports the [`concurrent`](/api/test#test-concurrent) option to run tests together. If this option is set, Vitest will group concurrent tests in the same _file_ (the number of simultaneously running tests depends on the [`maxConcurrency`](/config/maxconcurrency) option) and run them with [`Promise.all`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all). +The hook execution order within a single group is also controlled by [`sequence.hooks`](/config/sequence#sequence-hooks). With `sequence.hooks: 'parallel'`, the execution is bounded by the same limit of [`maxConcurrency`](/config/maxconcurrency). + Vitest doesn't perform any smart analysis and doesn't create additional workers to run these tests. This means that the performance of your tests will improve only if you rely heavily on asynchronous operations. For example, these tests will still run one after another even though the `concurrent` option is specified. This is because they are synchronous: ```ts diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 57688d9ad971..db1d5ae3f892 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -18,6 +18,7 @@ import type { TestContext, WriteableTestContext, } from './types/tasks' +import type { ConcurrencyLimiter } from './utils/limit-concurrency' import { processError } from '@vitest/utils/error' // TODO: load dynamically import { shuffle } from '@vitest/utils/helpers' import { getSafeTimers } from '@vitest/utils/timers' @@ -35,6 +36,7 @@ import { hasFailed, hasTests } from './utils/tasks' const now = globalThis.performance ? globalThis.performance.now.bind(globalThis.performance) : Date.now const unixNow = Date.now const { clearTimeout, setTimeout } = getSafeTimers() +let limitMaxConcurrency: ConcurrencyLimiter /** * Normalizes retry configuration to extract individual values. @@ -141,7 +143,7 @@ async function callTestHooks( if (sequence === 'parallel') { try { - await Promise.all(hooks.map(fn => fn(test.context))) + await Promise.all(hooks.map(fn => limitMaxConcurrency(() => fn(test.context)))) } catch (e) { failTask(test.result!, e, runner.config.diffOptions) @@ -150,7 +152,7 @@ async function callTestHooks( else { for (const fn of hooks) { try { - await fn(test.context) + await limitMaxConcurrency(() => fn(test.context)) } catch (e) { failTask(test.result!, e, runner.config.diffOptions) @@ -188,11 +190,13 @@ export async function callSuiteHook( } async function runHook(hook: Function) { - return getBeforeHookCleanupCallback( - hook, - await hook(...args), - name === 'beforeEach' ? args[0] as TestContext : undefined, - ) + return limitMaxConcurrency(async () => { + return getBeforeHookCleanupCallback( + hook, + await hook(...args), + name === 'beforeEach' ? args[0] as TestContext : undefined, + ) + }) } if (sequence === 'parallel') { @@ -311,6 +315,8 @@ async function callAroundHooks( let useCalled = false let setupTimeout: ReturnType let teardownTimeout: ReturnType | undefined + let setupLimitConcurrencyRelease: (() => void) | undefined + let teardownLimitConcurrencyRelease: (() => void) | undefined // Promise that resolves when use() is called (setup phase complete) let resolveUseCalled!: () => void @@ -352,10 +358,13 @@ async function callAroundHooks( // Setup phase completed - clear setup timer setupTimeout.clear() + setupLimitConcurrencyRelease?.() // Run inner hooks - don't time this against our teardown timeout await runNextHook(index + 1).catch(e => hookErrors.push(e)) + teardownLimitConcurrencyRelease = await limitMaxConcurrency.acquire() + // Start teardown timer after inner hooks complete - only times this hook's teardown code teardownTimeout = createTimeoutPromise(timeout, 'teardown', stackTraceError) @@ -363,6 +372,8 @@ async function callAroundHooks( resolveUseReturned() } + setupLimitConcurrencyRelease = await limitMaxConcurrency.acquire() + // Start setup timeout setupTimeout = createTimeoutPromise(timeout, 'setup', stackTraceError) @@ -381,6 +392,10 @@ async function callAroundHooks( catch (error) { rejectHookComplete(error as Error) } + finally { + setupLimitConcurrencyRelease?.() + teardownLimitConcurrencyRelease?.() + } })() // Wait for either: use() to be called OR hook to complete (error) OR setup timeout @@ -392,6 +407,7 @@ async function callAroundHooks( ]) } finally { + setupLimitConcurrencyRelease?.() setupTimeout.clear() } @@ -410,6 +426,7 @@ async function callAroundHooks( ]) } finally { + teardownLimitConcurrencyRelease?.() teardownTimeout?.clear() } } @@ -524,7 +541,7 @@ async function callCleanupHooks(runner: VitestRunner, cleanups: unknown[]) { if (typeof fn !== 'function') { return } - await fn() + await limitMaxConcurrency(() => fn()) }), ) } @@ -533,7 +550,7 @@ async function callCleanupHooks(runner: VitestRunner, cleanups: unknown[]) { if (typeof fn !== 'function') { continue } - await fn() + await limitMaxConcurrency(() => fn()) } } } @@ -623,7 +640,7 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { )) if (runner.runTask) { - await $('test.callback', () => runner.runTask!(test)) + await $('test.callback', () => limitMaxConcurrency(() => runner.runTask!(test))) } else { const fn = getFn(test) @@ -632,7 +649,7 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { 'Test function is not found. Did you add it using `setFn`?', ) } - await $('test.callback', () => fn()) + await $('test.callback', () => limitMaxConcurrency(() => fn())) } await runner.onAfterTryTask?.(test, { @@ -940,12 +957,10 @@ export async function runSuite(suite: Suite, runner: VitestRunner): Promise - async function runSuiteChild(c: Task, runner: VitestRunner) { const $ = runner.trace! if (c.type === 'test') { - return limitMaxConcurrency(() => $( + return $( 'run.test', { 'vitest.test.id': c.id, @@ -957,7 +972,7 @@ async function runSuiteChild(c: Task, runner: VitestRunner) { 'code.column.number': c.location?.column, }, () => runTest(c, runner), - )) + ) } else if (c.type === 'suite') { return $( diff --git a/packages/runner/src/utils/limit-concurrency.ts b/packages/runner/src/utils/limit-concurrency.ts index 4f42d5792774..d8ae7f2c2865 100644 --- a/packages/runner/src/utils/limit-concurrency.ts +++ b/packages/runner/src/utils/limit-concurrency.ts @@ -1,10 +1,16 @@ // A compact (code-wise, probably not memory-wise) singly linked list node. type QueueNode = [value: T, next?: QueueNode] +export interface ConcurrencyLimiter extends ConcurrencyLimiterFn { + acquire: () => (() => void) | Promise<() => void> +} + +type ConcurrencyLimiterFn = (func: (...args: Args) => PromiseLike | T, ...args: Args) => Promise + /** * Return a function for running multiple async operations with limited concurrency. */ -export function limitConcurrency(concurrency: number = Infinity): (func: (...args: Args) => PromiseLike | T, ...args: Args) => Promise { +export function limitConcurrency(concurrency: number = Infinity): ConcurrencyLimiter { // The number of currently active + pending tasks. let count = 0 @@ -30,28 +36,50 @@ export function limitConcurrency(concurrency: number = Infinity): { - // Create a promise chain that: - // 1. Waits for its turn in the task queue (if necessary). - // 2. Runs the task. - // 3. Allows the next pending task (if any) to run. - return new Promise((resolve) => { - if (count++ < concurrency) { - // No need to queue if fewer than maxConcurrency tasks are running. - resolve() + const acquire = () => { + let released = false + const release = () => { + if (!released) { + released = true + finish() } - else if (tail) { + } + + if (count++ < concurrency) { + return release + } + + return new Promise<() => void>((resolve) => { + if (tail) { // There are pending tasks, so append to the queue. - tail = tail[1] = [resolve] + tail = tail[1] = [() => resolve(release)] } else { // No other pending tasks, initialize the queue with a new tail and head. - head = tail = [resolve] + head = tail = [() => resolve(release)] } - }).then(() => { - // Running func here ensures that even a non-thenable result or an - // immediately thrown error gets wrapped into a Promise. - return func(...args) - }).finally(finish) + }) } + + const limiterFn: ConcurrencyLimiterFn = (func, ...args) => { + function run(release: () => void) { + try { + const result = func(...args) + if (result instanceof Promise) { + return result.finally(release) + } + release() + return Promise.resolve(result) + } + catch (error) { + release() + return Promise.reject(error) + } + } + + const release = acquire() + return release instanceof Promise ? release.then(run) : run(release) + } + + return Object.assign(limiterFn, { acquire }) } diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index 8ead70196400..b10a61856685 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -733,7 +733,7 @@ export const cliOptionsConfig: VitestCLIOptions = { }, }, maxConcurrency: { - description: 'Maximum number of concurrent tests in a suite (default: `5`)', + description: 'Maximum number of concurrent tests and suites during test file execution (default: `5`)', argument: '', }, expect: { diff --git a/test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts b/test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts deleted file mode 100644 index 7e49e748538b..000000000000 --- a/test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { createDefer } from '@vitest/utils/helpers' -import { describe, test, vi, expect } from 'vitest' - -// 3 tests depend on each other, -// so they will deadlock when maxConcurrency < 3 -// -// [a] [b] [c] -// * -> -// * -> -// <- * -// <------ - -vi.setConfig({ maxConcurrency: 2 }) - -describe('wrapper', { concurrent: true, timeout: 500 }, () => { - const defers = [ - createDefer(), - createDefer(), - createDefer(), - ] - - describe('1st suite', () => { - test('a', async () => { - expect(1).toBe(1) - defers[0].resolve() - await defers[2] - }) - - test('b', async () => { - expect(1).toBe(1) - await defers[0] - defers[1].resolve() - await defers[2] - }) - }) - - describe('2nd suite', () => { - test('c', async () => { - expect(1).toBe(1) - await defers[1] - defers[2].resolve() - }) - }) -}) diff --git a/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts b/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts deleted file mode 100644 index 851069137ed1..000000000000 --- a/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { describe, expect, test, vi } from 'vitest' -import { createDefer } from '@vitest/utils/helpers' - -// 3 tests depend on each other, -// so they will deadlock when maxConcurrency < 3 -// -// [a] [b] [c] -// * -> -// * -> -// <- * -// <------ - -vi.setConfig({ maxConcurrency: 2 }) - -describe('wrapper', { concurrent: true, timeout: 500 }, () => { - const defers = [ - createDefer(), - createDefer(), - createDefer(), - ] - - test('a', async () => { - expect(1).toBe(1) - defers[0].resolve() - await defers[2] - }) - - test('b', async () => { - expect(1).toBe(1) - await defers[0] - defers[1].resolve() - await defers[2] - }) - - test('c', async () => { - expect(1).toBe(1) - await defers[1] - defers[2].resolve() - }) -}) diff --git a/test/cli/test/__snapshots__/fails.test.ts.snap b/test/cli/test/__snapshots__/fails.test.ts.snap index 80a8c0ff3a45..f6db738593b8 100644 --- a/test/cli/test/__snapshots__/fails.test.ts.snap +++ b/test/cli/test/__snapshots__/fails.test.ts.snap @@ -7,16 +7,6 @@ exports[`should fail async-assertion.test.ts 1`] = ` AssertionError: expected 'xx' to be 'yy' // Object.is equality" `; -exports[`should fail concurrent-suite-deadlock.test.ts 1`] = ` -"Error: Test timed out in 500ms. -Error: Test timed out in 500ms." -`; - -exports[`should fail concurrent-test-deadlock.test.ts 1`] = ` -"Error: Test timed out in 500ms. -Error: Test timed out in 500ms." -`; - exports[`should fail each-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`; exports[`should fail empty.test.ts 1`] = `"Error: No test suite found in file /empty.test.ts"`; diff --git a/test/cli/test/around-each.test.ts b/test/cli/test/around-each.test.ts index e57d8e4748eb..247b64066fce 100644 --- a/test/cli/test/around-each.test.ts +++ b/test/cli/test/around-each.test.ts @@ -1064,6 +1064,48 @@ test('aroundEach hook timeouts are independent of each other', async () => { `) }) +test('aroundEach teardown timeout works when runTest error is caught', async () => { + const { errorTree } = await runInlineTests({ + 'caught-inner-error-timeout.test.ts': ` + import { aroundEach, describe, expect, test } from 'vitest' + + describe('suite', () => { + aroundEach(async (runTest) => { + try { + await runTest() + } + catch { + // swallow inner hook failure, then run teardown work + } + await new Promise(resolve => setTimeout(resolve, 200)) + }, 50) + + aroundEach(async (runTest) => { + await runTest() + throw new Error('inner aroundEach teardown failure') + }) + + test('test', () => { + expect(1).toBe(1) + }) + }) + `, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "caught-inner-error-timeout.test.ts": { + "suite": { + "test": [ + "inner aroundEach teardown failure", + "The teardown phase of "aroundEach" hook timed out after 50ms.", + ], + }, + }, + } + `) +}) + test('aroundEach with AsyncLocalStorage', async () => { const { stdout, stderr, errorTree } = await runInlineTests({ 'async-local-storage.test.ts': ` @@ -1633,6 +1675,49 @@ test('aroundAll receives suite as third argument', async () => { `) }) +test('aroundAll teardown timeout works when runSuite error is caught', async () => { + const { errorTree } = await runInlineTests({ + 'caught-inner-suite-error-timeout.test.ts': ` + import { aroundAll, describe, expect, test } from 'vitest' + + describe('suite', () => { + aroundAll(async (runSuite) => { + try { + await runSuite() + } + catch { + // swallow inner hook failure, then run teardown work + } + await new Promise(resolve => setTimeout(resolve, 200)) + }, 50) + + aroundAll(async (runSuite) => { + await runSuite() + throw new Error('inner aroundAll teardown failure') + }) + + test('test', () => { + expect(1).toBe(1) + }) + }) + `, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "caught-inner-suite-error-timeout.test.ts": { + "suite": { + "__suite_errors__": [ + "inner aroundAll teardown failure", + "The teardown phase of "aroundAll" hook timed out after 50ms.", + ], + "test": "passed", + }, + }, + } + `) +}) + test('aroundAll with server start/stop pattern', async () => { const { stdout, stderr, errorTree } = await runInlineTests({ 'server.test.ts': ` @@ -2043,6 +2128,10 @@ test('aroundAll teardown timeout works when inner fails', async () => { { "caught-inner-error-timeout.test.ts": { "suite": { + "__suite_errors__": [ + "inner aroundAll teardown failure", + "The teardown phase of "aroundAll" hook timed out after 50ms.", + ], "test": "passed", }, }, @@ -2181,6 +2270,9 @@ test('aroundAll aborts late runSuite after setup timeout', async () => { { "late-run-suite-after-timeout.test.ts": { "timed out suite": { + "__suite_errors__": [ + "The setup phase of "aroundAll" hook timed out after 10ms.", + ], "basic": "skipped", }, }, @@ -2228,7 +2320,7 @@ test('nested aroundEach setup error is not propagated to outer runTest catch', a | ^ 18| }) 19| - ❯ nested-around-each-setup-error.test.ts:7:17 + ❯ nested-around-each-setup-error.test.ts:7:11 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ @@ -2355,7 +2447,7 @@ test('nested aroundAll setup error is not propagated to outer runSuite catch', a | ^ 18| }) 19| - ❯ nested-around-all-setup-error.test.ts:7:17 + ❯ nested-around-all-setup-error.test.ts:7:11 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ diff --git a/test/cli/test/concurrent.test.ts b/test/cli/test/concurrent.test.ts new file mode 100644 index 000000000000..80601067a554 --- /dev/null +++ b/test/cli/test/concurrent.test.ts @@ -0,0 +1,1076 @@ +import { expect, test } from 'vitest' +import { runInlineTests } from '../../test-utils' + +// 3 tests depend on each other, +// so they will deadlock when maxConcurrency < 3 +// +// [a] [b] [c] +// * -> +// * -> +// <- * +// <------ + +const deadlockSource = ` +import { describe, expect, test } from 'vitest' +import { createDefer } from '@vitest/utils/helpers' + +describe.concurrent('wrapper', () => { + const defers = [ + createDefer(), + createDefer(), + createDefer(), + ] + + test('a', async () => { + expect(1).toBe(1) + defers[0].resolve() + await defers[2] + }) + + test('b', async () => { + expect(1).toBe(1) + await defers[0] + defers[1].resolve() + await defers[2] + }) + + test('c', async () => { + expect(1).toBe(1) + await defers[1] + defers[2].resolve() + }) +}) +` + +test('deadlocks with insufficient maxConcurrency', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': deadlockSource, + }, { + maxConcurrency: 2, + testTimeout: 500, + }) + + // "a" and "b" fill both concurrency slots and wait for `defers[2]`. + // "c" is queued until one slot is released by timeout, then it starts, + // observes `defers[1]` already resolved by "b", resolves `defers[2]`, and passes. + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "a": [ + "Test timed out in 500ms. + If this is a long-running test, pass a timeout value as the last argument or configure it globally with "testTimeout".", + ], + "b": [ + "Test timed out in 500ms. + If this is a long-running test, pass a timeout value as the last argument or configure it globally with "testTimeout".", + ], + "c": "passed", + }, + }, + } + `) +}) + +test('passes when maxConcurrency is high enough', async () => { + const { stderr, errorTree } = await runInlineTests({ + 'basic.test.ts': deadlockSource, + }, { + maxConcurrency: 3, + }) + + expect(stderr).toBe('') + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "a": "passed", + "b": "passed", + "c": "passed", + }, + }, + } + `) +}) + +const suiteDeadlockSource = ` +import { describe, expect, test } from 'vitest' +import { createDefer } from '@vitest/utils/helpers' + +describe.concurrent('wrapper', () => { + const defers = [ + createDefer(), + createDefer(), + createDefer(), + ] + + describe('1st suite', () => { + test('a', async () => { + expect(1).toBe(1) + defers[0].resolve() + await defers[2] + }) + + test('b', async () => { + expect(1).toBe(1) + await defers[0] + defers[1].resolve() + await defers[2] + }) + }) + + describe('2nd suite', () => { + test('c', async () => { + expect(1).toBe(1) + await defers[1] + defers[2].resolve() + }) + }) +}) +` + +test('suite deadlocks with insufficient maxConcurrency', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': suiteDeadlockSource, + }, { + maxConcurrency: 2, + testTimeout: 500, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "1st suite": { + "a": [ + "Test timed out in 500ms. + If this is a long-running test, pass a timeout value as the last argument or configure it globally with "testTimeout".", + ], + "b": [ + "Test timed out in 500ms. + If this is a long-running test, pass a timeout value as the last argument or configure it globally with "testTimeout".", + ], + }, + "2nd suite": { + "c": "passed", + }, + }, + }, + } + `) +}) + +test('suite passes when maxConcurrency is high enough', async () => { + const { stderr, errorTree } = await runInlineTests({ + 'basic.test.ts': suiteDeadlockSource, + }, { + maxConcurrency: 3, + }) + + expect(stderr).toBe('') + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "1st suite": { + "a": "passed", + "b": "passed", + }, + "2nd suite": { + "c": "passed", + }, + }, + }, + } + `) +}) + +const beforeAllNeighboringSuitesSource = ` +import { beforeAll, describe, expect, test } from 'vitest' +import { createDefer } from '@vitest/utils/helpers' + +const defers = [ + createDefer(), + createDefer(), +] + +describe.concurrent('s1', () => { + beforeAll(async () => { + defers[0].resolve() + await defers[1] + }) + + test('a', () => { + expect(1).toBe(1) + }) +}) + +describe.concurrent('s2', () => { + beforeAll(async () => { + await defers[0] + defers[1].resolve() + }) + + test('b', () => { + expect(1).toBe(1) + }) +}) +` + +test('neighboring suite beforeAll deadlocks with insufficient maxConcurrency', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': beforeAllNeighboringSuitesSource, + }, { + maxConcurrency: 1, + hookTimeout: 500, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "s1": { + "__suite_errors__": [ + "Hook timed out in 500ms. + If this is a long-running hook, pass a timeout value as the last argument or configure it globally with "hookTimeout".", + ], + "a": "skipped", + }, + "s2": { + "b": "passed", + }, + }, + } + `) +}) + +test('neighboring suite beforeAll passes when maxConcurrency is high enough', async () => { + const { stderr, errorTree } = await runInlineTests({ + 'basic.test.ts': beforeAllNeighboringSuitesSource, + }, { + maxConcurrency: 2, + }) + + expect(stderr).toBe('') + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "s1": { + "a": "passed", + }, + "s2": { + "b": "passed", + }, + }, + } + `) +}) + +const afterAllNeighboringSuitesSource = ` +import { afterAll, describe, expect, test } from 'vitest' +import { createDefer } from '@vitest/utils/helpers' + +const defers = [ + createDefer(), + createDefer(), +] + +describe.concurrent('s1', () => { + afterAll(async () => { + defers[0].resolve() + await defers[1] + }) + + test('a', () => { + expect(1).toBe(1) + }) +}) + +describe.concurrent('s2', () => { + afterAll(async () => { + await defers[0] + defers[1].resolve() + }) + + test('b', () => { + expect(1).toBe(1) + }) +}) +` + +test('neighboring suite afterAll deadlocks with insufficient maxConcurrency', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': afterAllNeighboringSuitesSource, + }, { + maxConcurrency: 1, + hookTimeout: 500, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "s1": { + "__suite_errors__": [ + "Hook timed out in 500ms. + If this is a long-running hook, pass a timeout value as the last argument or configure it globally with "hookTimeout".", + ], + "a": "passed", + }, + "s2": { + "b": "passed", + }, + }, + } + `) +}) + +test('neighboring suite afterAll passes when maxConcurrency is high enough', async () => { + const { stderr, errorTree } = await runInlineTests({ + 'basic.test.ts': afterAllNeighboringSuitesSource, + }, { + maxConcurrency: 2, + }) + + expect(stderr).toBe('') + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "s1": { + "a": "passed", + }, + "s2": { + "b": "passed", + }, + }, + } + `) +}) + +const beforeEachDeadlockSource = ` +import { beforeEach, describe, expect, test } from 'vitest' +import { createDefer } from '@vitest/utils/helpers' + +describe.concurrent('wrapper', () => { + const defers = [ + createDefer(), + createDefer(), + createDefer(), + ] + + beforeEach(async () => { + defers[0].resolve() + await defers[2] + }) + + beforeEach(async () => { + await defers[0] + defers[1].resolve() + await defers[2] + }) + + beforeEach(async () => { + await defers[1] + defers[2].resolve() + }) + + test('t', () => { + expect(1).toBe(1) + }) +}) +` + +test('beforeEach deadlocks with insufficient maxConcurrency', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': beforeEachDeadlockSource, + }, { + maxConcurrency: 2, + sequence: { hooks: 'parallel' }, + hookTimeout: 500, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "t": [ + "Hook timed out in 500ms. + If this is a long-running hook, pass a timeout value as the last argument or configure it globally with "hookTimeout".", + ], + }, + }, + } + `) +}) + +test('beforeEach passes when maxConcurrency is high enough', async () => { + const { stderr, errorTree } = await runInlineTests({ + 'basic.test.ts': beforeEachDeadlockSource, + }, { + maxConcurrency: 3, + sequence: { hooks: 'parallel' }, + }) + + expect(stderr).toBe('') + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "t": "passed", + }, + }, + } + `) +}) + +const afterEachDeadlockSource = ` +import { afterEach, describe, expect, test } from 'vitest' +import { createDefer } from '@vitest/utils/helpers' + +describe.concurrent('wrapper', () => { + const defers = [ + createDefer(), + createDefer(), + createDefer(), + ] + + afterEach(async () => { + defers[0].resolve() + await defers[2] + }) + + afterEach(async () => { + await defers[0] + defers[1].resolve() + await defers[2] + }) + + afterEach(async () => { + await defers[1] + defers[2].resolve() + }) + + test('t', () => { + expect(1).toBe(1) + }) +}) +` + +test('afterEach deadlocks with insufficient maxConcurrency', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': afterEachDeadlockSource, + }, { + maxConcurrency: 2, + sequence: { hooks: 'parallel' }, + hookTimeout: 500, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "t": [ + "Hook timed out in 500ms. + If this is a long-running hook, pass a timeout value as the last argument or configure it globally with "hookTimeout".", + ], + }, + }, + } + `) +}) + +test('afterEach passes when maxConcurrency is high enough', async () => { + const { stderr, errorTree } = await runInlineTests({ + 'basic.test.ts': afterEachDeadlockSource, + }, { + maxConcurrency: 3, + sequence: { hooks: 'parallel' }, + }) + + expect(stderr).toBe('') + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "t": "passed", + }, + }, + } + `) +}) + +const aroundAllNeighboringSuitesSource = ` +import { aroundAll, describe, expect, test } from 'vitest' +import { createDefer } from '@vitest/utils/helpers' + +const defers = [ + createDefer(), + createDefer(), +] + +describe.concurrent('s1', () => { + aroundAll(async (runSuite) => { + defers[0].resolve() + await defers[1] + await runSuite() + }) + + test('a', () => { + expect(1).toBe(1) + }) +}) + +describe.concurrent('s2', () => { + aroundAll(async (runSuite) => { + await defers[0] + defers[1].resolve() + await runSuite() + }) + + test('b', () => { + expect(1).toBe(1) + }) +}) +` + +test('neighboring suite aroundAll deadlocks with insufficient maxConcurrency', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': aroundAllNeighboringSuitesSource, + }, { + maxConcurrency: 1, + hookTimeout: 500, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "s1": { + "__suite_errors__": [ + "The setup phase of "aroundAll" hook timed out after 500ms.", + ], + "a": "skipped", + }, + "s2": { + "b": "passed", + }, + }, + } + `) +}) + +test('neighboring suite aroundAll passes when maxConcurrency is high enough', async () => { + const { stderr, errorTree } = await runInlineTests({ + 'basic.test.ts': aroundAllNeighboringSuitesSource, + }, { + maxConcurrency: 2, + }) + + expect(stderr).toBe('') + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "s1": { + "a": "passed", + }, + "s2": { + "b": "passed", + }, + }, + } + `) +}) + +const aroundAllNeighboringSuitesPostSource = ` +import { aroundAll, describe, expect, test } from 'vitest' +import { createDefer } from '@vitest/utils/helpers' + +const defers = [ + createDefer(), + createDefer(), +] + +describe.concurrent('s1', () => { + aroundAll(async (runSuite) => { + await runSuite() + defers[0].resolve() + await defers[1] + }) + + test('a', () => { + expect(1).toBe(1) + }) +}) + +describe.concurrent('s2', () => { + aroundAll(async (runSuite) => { + await runSuite() + await defers[0] + defers[1].resolve() + }) + + test('b', () => { + expect(1).toBe(1) + }) +}) +` + +test('neighboring suite aroundAll teardown deadlocks with insufficient maxConcurrency', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': aroundAllNeighboringSuitesPostSource, + }, { + maxConcurrency: 1, + hookTimeout: 500, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "s1": { + "__suite_errors__": [ + "The teardown phase of \"aroundAll\" hook timed out after 500ms.", + ], + "a": "passed", + }, + "s2": { + "b": "passed", + }, + }, + } + `) +}) + +test('neighboring suite aroundAll teardown passes when maxConcurrency is high enough', async () => { + const { stderr, errorTree } = await runInlineTests({ + 'basic.test.ts': aroundAllNeighboringSuitesPostSource, + }, { + maxConcurrency: 2, + }) + + expect(stderr).toBe('') + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "s1": { + "a": "passed", + }, + "s2": { + "b": "passed", + }, + }, + } + `) +}) + +const aroundAllSetupTimeoutLateTeardownAcquireSource = ` +import { aroundAll, describe, expect, test } from 'vitest' +import { createDefer } from '@vitest/utils/helpers' + +const unblockS1Setup = createDefer() +const allowS2TestFinish = createDefer() +const blockForever = createDefer() + +describe.concurrent('s1', () => { + aroundAll(async (runSuite) => { + await unblockS1Setup + await runSuite() + allowS2TestFinish.resolve() + await blockForever + }) + + test('a', () => { + expect(1).toBe(1) + }) +}) + +describe.concurrent('s2', () => { + aroundAll(async (runSuite) => { + unblockS1Setup.resolve() + await runSuite() + }) + + test('b', async () => { + await allowS2TestFinish + expect(1).toBe(1) + }) +}) +` + +test('neighboring suite aroundAll does not hang when setup times out before late teardown acquire', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': aroundAllSetupTimeoutLateTeardownAcquireSource, + }, { + maxConcurrency: 1, + hookTimeout: 500, + testTimeout: 500, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "s1": { + "__suite_errors__": [ + "The setup phase of "aroundAll" hook timed out after 500ms.", + ], + "a": "skipped", + }, + "s2": { + "b": [ + "Test timed out in 500ms. + If this is a long-running test, pass a timeout value as the last argument or configure it globally with "testTimeout".", + ], + }, + }, + } + `) +}) + +const aroundEachNeighboringTestsSource = ` +import { aroundEach, describe, expect, test } from 'vitest' +import { createDefer } from '@vitest/utils/helpers' + +const defers = [ + createDefer(), + createDefer(), +] + +describe.concurrent('wrapper', () => { + aroundEach(async (runTest, context) => { + if (context.task.name === 'a') { + defers[0].resolve() + await defers[1] + await runTest() + return + } + + await defers[0] + defers[1].resolve() + await runTest() + }) + + test('a', () => { + expect(1).toBe(1) + }) + + test('b', () => { + expect(1).toBe(1) + }) +}) +` + +test('neighboring test aroundEach deadlocks with insufficient maxConcurrency', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': aroundEachNeighboringTestsSource, + }, { + maxConcurrency: 1, + hookTimeout: 500, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "a": [ + "The setup phase of \"aroundEach\" hook timed out after 500ms.", + ], + "b": "passed", + }, + }, + } + `) +}) + +test('neighboring test aroundEach passes when maxConcurrency is high enough', async () => { + const { stderr, errorTree } = await runInlineTests({ + 'basic.test.ts': aroundEachNeighboringTestsSource, + }, { + maxConcurrency: 2, + }) + + expect(stderr).toBe('') + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "a": "passed", + "b": "passed", + }, + }, + } + `) +}) + +const aroundEachNeighboringTestsPostSource = ` +import { aroundEach, describe, expect, test } from 'vitest' +import { createDefer } from '@vitest/utils/helpers' + +const defers = [ + createDefer(), + createDefer(), +] + +describe.concurrent('wrapper', () => { + aroundEach(async (runTest, context) => { + await runTest() + + if (context.task.name === 'a') { + defers[0].resolve() + await defers[1] + return + } + + await defers[0] + defers[1].resolve() + }) + + test('a', () => { + expect(1).toBe(1) + }) + + test('b', () => { + expect(1).toBe(1) + }) +}) +` + +test('neighboring test aroundEach teardown deadlocks with insufficient maxConcurrency', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': aroundEachNeighboringTestsPostSource, + }, { + maxConcurrency: 1, + hookTimeout: 500, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "a": [ + "The teardown phase of \"aroundEach\" hook timed out after 500ms.", + ], + "b": "passed", + }, + }, + } + `) +}) + +test('neighboring test aroundEach teardown passes when maxConcurrency is high enough', async () => { + const { stderr, errorTree } = await runInlineTests({ + 'basic.test.ts': aroundEachNeighboringTestsPostSource, + }, { + maxConcurrency: 2, + }) + + expect(stderr).toBe('') + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "a": "passed", + "b": "passed", + }, + }, + } + `) +}) + +const aroundEachOuterCatchesInnerErrorSource = ` +import { aroundEach, describe, expect, test } from 'vitest' + +describe.concurrent('wrapper', () => { + aroundEach(async (runTest) => { + let runTestError: unknown + try { + await runTest() + } + catch (error) { + runTestError = error + } + + await Promise.resolve() + + if (runTestError) { + throw runTestError + } + }) + + aroundEach(async (runTest, context) => { + await runTest() + if (context.task.name === 'a') { + throw new Error('inner aroundEach teardown failure') + } + }) + + test('a', () => { + expect(1).toBe(1) + }) + + test('b', () => { + expect(1).toBe(1) + }) +}) +` + +test('aroundEach continues protocol when outer hook catches runTest error', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': aroundEachOuterCatchesInnerErrorSource, + }, { + maxConcurrency: 1, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "a": [ + "inner aroundEach teardown failure", + ], + "b": "passed", + }, + }, + } + `) +}) + +const aroundAllOuterCatchesInnerErrorSource = ` +import { aroundAll, describe, expect, test } from 'vitest' + +describe.concurrent('suite', () => { + aroundAll(async (runSuite) => { + let runSuiteError: unknown + try { + await runSuite() + } + catch (error) { + runSuiteError = error + } + + await Promise.resolve() + + if (runSuiteError) { + throw runSuiteError + } + }) + + aroundAll(async (runSuite) => { + await runSuite() + throw new Error('inner aroundAll teardown failure') + }) + + test('a', () => { + expect(1).toBe(1) + }) +}) +` + +test('aroundAll continues protocol when outer hook catches runSuite error', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': aroundAllOuterCatchesInnerErrorSource, + }, { + maxConcurrency: 1, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "suite": { + "__suite_errors__": [ + "inner aroundAll teardown failure", + ], + "a": "passed", + }, + }, + } + `) +}) + +const aroundEachCaughtInnerErrorTeardownTimeoutSource = ` +import { aroundEach, describe, expect, test } from 'vitest' + +describe.concurrent('wrapper', () => { + aroundEach(async (runTest) => { + try { + await runTest() + } + catch { + // swallow inner failure, then run long teardown logic + } + await new Promise(resolve => setTimeout(resolve, 200)) + }, 50) + + aroundEach(async (runTest) => { + await runTest() + throw new Error('inner aroundEach teardown failure') + }) + + test('a', () => { + expect(1).toBe(1) + }) +}) +` + +test('aroundEach enforces teardown timeout when inner error is caught', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': aroundEachCaughtInnerErrorTeardownTimeoutSource, + }, { + maxConcurrency: 1, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "wrapper": { + "a": [ + "inner aroundEach teardown failure", + "The teardown phase of "aroundEach" hook timed out after 50ms.", + ], + }, + }, + } + `) +}) + +const aroundAllCaughtInnerErrorTeardownTimeoutSource = ` +import { aroundAll, describe, expect, test } from 'vitest' + +describe.concurrent('suite', () => { + aroundAll(async (runSuite) => { + try { + await runSuite() + } + catch { + // swallow inner failure, then run long teardown logic + } + await new Promise(resolve => setTimeout(resolve, 200)) + }, 50) + + aroundAll(async (runSuite) => { + await runSuite() + throw new Error('inner aroundAll teardown failure') + }) + + test('a', () => { + expect(1).toBe(1) + }) +}) +` + +test('aroundAll enforces teardown timeout when inner error is caught', async () => { + const { errorTree } = await runInlineTests({ + 'basic.test.ts': aroundAllCaughtInnerErrorTeardownTimeoutSource, + }, { + maxConcurrency: 1, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "basic.test.ts": { + "suite": { + "__suite_errors__": [ + "inner aroundAll teardown failure", + "The teardown phase of "aroundAll" hook timed out after 50ms.", + ], + "a": "passed", + }, + }, + } + `) +}) diff --git a/test/cli/test/expect-task.test.ts b/test/cli/test/expect-task.test.ts index d1dae4edb203..a017bfc5a133 100644 --- a/test/cli/test/expect-task.test.ts +++ b/test/cli/test/expect-task.test.ts @@ -172,13 +172,15 @@ describe('serial', { concurrent: true }, () => { name: 'test-bound extend & local extend', test: testBoundLocalExtend, }, - ] as const)('works with $name', async ({ options, test }, { expect }) => { + ] as const)('works with $name', async ({ options, test }, { task, expect }) => { const { stdout, stderr } = await runInlineTests( { 'basic.test.ts': test, 'to-match-test.ts': toMatchTest, }, { reporters: ['tap'], ...options }, + undefined, + task, ) expect(stderr).toBe('') @@ -210,13 +212,15 @@ describe('concurrent', { concurrent: true }, () => { name: 'global import', test: withConcurrency(globalImport), }, - ] as const)('fails with $name', async ({ options, test }, { expect }) => { + ] as const)('fails with $name', async ({ options, test }, { task, expect }) => { const { stdout, ctx } = await runInlineTests( { 'basic.test.ts': test, 'to-match-test.ts': toMatchTest, }, { reporters: ['tap'], ...options }, + undefined, + task, ) expect( @@ -263,13 +267,15 @@ describe('concurrent', { concurrent: true }, () => { name: 'test-bound extend & local extend', test: withConcurrency(testBoundLocalExtend), }, - ])('works with $name', async ({ test }, { expect }) => { + ])('works with $name', async ({ test }, { task, expect }) => { const { stdout } = await runInlineTests( { 'basic.test.ts': test, 'to-match-test.ts': toMatchTest, }, { reporters: ['tap'] }, + undefined, + task, ) expect(stdout.replace(/[\d.]+m?s/g, '