Skip to content

feat: first commit flaky #61746

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
vespa7 wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: first commit flaky #61746

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion lib/internal/test_runner/harness.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ function createTestTree(rootTestOptions, globalOptions) {
failed: 0,
passed: 0,
cancelled: 0,
flaky: 0,
skipped: 0,
todo: 0,
topLevel: 0,
Expand Down Expand Up @@ -377,7 +378,7 @@ function runInParentContext(Factory) {

return run(name, options, fn, overrides);
};
ArrayPrototypeForEach(['expectFailure', 'skip', 'todo', 'only'], (keyword) => {
ArrayPrototypeForEach(['expectFailure', 'flaky', 'skip', 'todo', 'only'], (keyword) => {
test[keyword] = (name, options, fn) => {
const overrides = {
__proto__: null,
Expand Down
10 changes: 7 additions & 3 deletions lib/internal/test_runner/reporter/tap.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,12 @@ async function * tapReporter(source) {
for await (const { type, data } of source) {
switch (type) {
case 'test:fail': {
yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure);
yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky);
const location = data.file ? `${data.file}:${data.line}:${data.column}` : null;
yield reportDetails(data.nesting, data.details, location);
break;
} case 'test:pass':
yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure);
yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky);
yield reportDetails(data.nesting, data.details, null);
break;
case 'test:plan':
Expand All @@ -65,7 +65,7 @@ async function * tapReporter(source) {
}
}

function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure) {
function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure, flaky) {
let line = `${indent(nesting)}${status} ${testNumber}`;

if (name) {
Expand All @@ -78,6 +78,10 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure
line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`;
} else if (expectFailure !== undefined) {
line += ' # EXPECTED FAILURE';
//should we use flaky >=0 here? for always printing 0 retries
} else if (flaky !== undefined && flaky > 0) {
const retryText = flaky === 1 ? 're-try' : 're-tries';
line += ` # FLAKY ${flaky} ${retryText}`;
Comment on lines 68 to +84
Copy link
Member

@JakobJingleheimer JakobJingleheimer Feb 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the feature proposal, the output cites the number of re-tries it took to succeed. So I think the name for flaky should be changed to something like flakyRetriedCount to disambiguate it with the maximum number of re-tries to attempt.

# FLAKY 42 re-tries

}

line += '\n';
Expand Down
5 changes: 4 additions & 1 deletion lib/internal/test_runner/reporter/utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ function formatError(error, indent) {
function formatTestReport(type, data, showErrorDetails = true, prefix = '', indent = '') {
let color = reporterColorMap[type] ?? colors.white;
let symbol = reporterUnicodeSymbolMap[type] ?? ' ';
const { skip, todo, expectFailure } = data;
const { skip, todo, expectFailure, flaky } = data;
const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : '';
let title = `${data.name}${duration_ms}`;

Expand All @@ -87,6 +87,9 @@ function formatTestReport(type, data, showErrorDetails = true, prefix = '', inde
}
} else if (expectFailure !== undefined) {
title += ` # EXPECTED FAILURE`;
} else if (flaky !== undefined && flaky > 0) {
const retryText = flaky === 1 ? 're-try' : 're-tries';
title += ` # FLAKY ${flaky} ${retryText}`;
Comment on lines 74 to +92
Copy link
Member

@JakobJingleheimer JakobJingleheimer Feb 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same (RE var name)

}

const err = showErrorDetails && data.details?.error ? formatError(data.details.error, indent) : '';
Expand Down
4 changes: 3 additions & 1 deletion lib/internal/test_runner/test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ const kTestTimeoutFailure = 'testTimeoutFailure';
const kExpectedFailure = 'expectedFailure';
const kHookFailure = 'hookFailed';
const kDefaultTimeout = null;
const kDefaultFlakyRetries = 20;
const noop = FunctionPrototype;
const kShouldAbort = Symbol('kShouldAbort');
const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']);
Expand Down Expand Up @@ -497,7 +498,8 @@ class Test extends AsyncResource {
super('Test');

let { fn, name, parent } = options;
const { concurrency, entryFile, expectFailure, loc, only, timeout, todo, skip, signal, plan } = options;

const { concurrency, entryFile, expectFailure, flaky, loc, only, timeout, todo, skip, signal, plan } = options;
Copy link
Member

@JakobJingleheimer JakobJingleheimer Feb 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: this is getting really long

Suggested change
const { concurrency, entryFile, expectFailure, flaky, loc, only, timeout, todo, skip, signal, plan } = options;
const {
entryFile,
expectFailure,
flaky,
loc,
only,
plan,
signal,
skip,
timeout,
todo,
concurrency,
} = options;


if (typeof fn !== 'function') {
fn = noop;
Expand Down
4 changes: 4 additions & 0 deletions lib/internal/test_runner/utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,10 @@ function countCompletedTest(test, harness = test.root.harness) {
} else {
harness.counters.passed++;
}

if (test.flakyRetries > 0) {
harness.counters.flaky++;
}
harness.counters.tests++;
}

Expand Down
6 changes: 6 additions & 0 deletions package-lock.json
View file
Open in desktop
Copy link
Member

@JakobJingleheimer JakobJingleheimer Feb 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this should not be committed?

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading