diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 1e41240bc2..221bb8b707 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -56,7 +56,7 @@ jobs: run: npm run build - name: Test - run: npm run test:unit -- -h -b ${{ matrix.BROWSER }} --jquery ${{ matrix.JQUERY }} + run: npm run test:unit -- -h -b ${{ matrix.BROWSER }} --jquery ${{ matrix.JQUERY }} --retries 3 edge: runs-on: windows-latest @@ -94,7 +94,7 @@ jobs: run: npm run build - name: Test - run: npm run test:unit -- -h -b edge --jquery ${{ matrix.JQUERY }} + run: npm run test:unit -- -h -b edge --jquery ${{ matrix.JQUERY }} --retries 3 safari: runs-on: macos-latest @@ -132,7 +132,7 @@ jobs: run: npm run build - name: Test - run: npm run test:unit -- -b safari --jquery ${{ matrix.JQUERY }} + run: npm run test:unit -- -b safari --jquery ${{ matrix.JQUERY }} --retries 3 legacy-build: runs-on: ubuntu-latest diff --git a/tests/runner/command.js b/tests/runner/command.js index b12729082a..655024fb4c 100644 --- a/tests/runner/command.js +++ b/tests/runner/command.js @@ -58,6 +58,11 @@ const argv = yargs( process.argv.slice( 2 ) ) "Leave the browser open for debugging. Cannot be used with --headless.", conflicts: [ "headless" ] } ) + .option( "retries", { + alias: "r", + type: "number", + description: "Number of times to retry failed tests." + } ) .option( "concurrency", { alias: "c", type: "number", diff --git a/tests/runner/run.js b/tests/runner/run.js index 877bed1baa..bf3a161912 100644 --- a/tests/runner/run.js +++ b/tests/runner/run.js @@ -7,7 +7,7 @@ import { generateHash } from "./lib/generateHash.js"; import { getBrowserString } from "./lib/getBrowserString.js"; import { suites as allSuites } from "./suites.js"; import { cleanupAllBrowsers, touchBrowser } from "./selenium/browsers.js"; -import { addRun, getNextBrowserTest, runAll } from "./selenium/queue.js"; +import { addRun, getNextBrowserTest, retryTest, runAll } from "./selenium/queue.js"; const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000; @@ -21,6 +21,7 @@ export async function run( { headless, jquery: jquerys = [], migrate, + retries = 0, suite: suites = [], verbose } ) { @@ -88,6 +89,13 @@ export async function run( { // Handle failure if ( failed ) { + const retry = retryTest( reportId, retries ); + + // Retry if retryTest returns a test + if ( retry ) { + return retry; + } + errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) ); } diff --git a/tests/runner/selenium/queue.js b/tests/runner/selenium/queue.js index c5a1596fd6..de24c5bb0f 100644 --- a/tests/runner/selenium/queue.js +++ b/tests/runner/selenium/queue.js @@ -1,3 +1,4 @@ +import chalk from "chalk"; import { getBrowserString } from "../lib/getBrowserString.js"; import { checkLastTouches, @@ -34,11 +35,30 @@ export function getNextBrowserTest( reportId ) { } } +export function retryTest( reportId, maxRetries ) { + if ( !maxRetries ) { + return; + } + const test = queue.find( ( test ) => test.id === reportId ); + if ( test ) { + test.retries++; + if ( test.retries <= maxRetries ) { + console.log( + `\nRetrying test ${ reportId } for ${ chalk.yellow( test.options.suite ) }...${ + test.retries + }` + ); + return test; + } + } +} + export function addRun( url, browser, options ) { queue.push( { browser, fullBrowser: getBrowserString( browser ), id: options.reportId, + retries: 0, url, options, running: false