feat: add support for test retries [sc-20570]

packages/cli/e2e/__tests__/fixtures/retry-project/checkly.config.ts

@@ -0,0 +1,22 @@
+import { defineConfig } from 'checkly'
+
+const config = defineConfig({
+  projectName: 'Test Project',
+  logicalId: 'test-project',
+  repoUrl: 'https://github.com/checkly/checkly-cli',
+  checks: {
+    locations: ['us-east-1', 'eu-west-1'],
+    tags: ['mac'],
+    runtimeId: '2023.09',
+    checkMatch: '**/*.check.ts',
+    browserChecks: {
+      // using .test.ts suffix (no .spec.ts) to avoid '@playwright/test not found error' when Jest transpile the spec.ts
+      testMatch: '**/__checks__/*.test.ts',
+    },
+  },
+  cli: {
+    runLocation: 'us-east-1',
+  },
+})
+
+export default config

packages/cli/e2e/__tests__/fixtures/retry-project/src/__checks__/group.check.ts

@@ -0,0 +1,25 @@
+import { CheckGroup, BrowserCheck } from 'checkly/constructs'
+
+const group = new CheckGroup('check-group-1', {
+  name: 'Group',
+  activated: true,
+  muted: false,
+  locations: ['us-east-1', 'eu-west-1'],
+  tags: ['mac', 'group'],
+  environmentVariables: [],
+  apiCheckDefaults: {},
+  alertChannels: [],
+  browserChecks: {
+    // using .test.ts suffix (no .spec.ts) to avoid '@playwright/test not found error' when Jest transpile the spec.ts
+    testMatch: '**/*.test.ts',
+  },
+})
+
+const browserCheck = new BrowserCheck('group-browser-check-1', {
+  name: 'Check with group',
+  activated: false,
+  groupId: group.ref(),
+  code: {
+    content: 'throw new Error("Failing Check Result")',
+  },
+})

packages/cli/e2e/__tests__/test.spec.ts

@@ -191,4 +191,17 @@ describe('test', () => {
       fs.rmSync(snapshotDir, { recursive: true })
     }
   })
+
+  it('Should execute retries', async () => {
+    const result = await runChecklyCli({
+      args: ['test', '--retries=3'],
+      apiKey: config.get('apiKey'),
+      accountId: config.get('accountId'),
+      directory: path.join(__dirname, 'fixtures', 'retry-project'),
+      timeout: 120000, // 2 minutes
+    })
+    // The failing check result will have "Failing Check Result" in the output.
+    // We expect the check to be run 4 times.
+    expect(result.stdout.match(/Failing Check Result/g)).toHaveLength(4)
+  })
 })

packages/cli/e2e/run-checkly.ts

@@ -33,7 +33,9 @@ export function runChecklyCli (options: {
         CHECKLY_API_KEY: apiKey,
         CHECKLY_ACCOUNT_ID: accountId,
         CHECKLY_ENV: process.env.CHECKLY_ENV,
-        CHECKLY_CLI_VERSION: cliVersion,
+        // We need the CLI to report 4.8.0 or greater in order for the backend to use the new MQTT topic format.
+        // Once 4.8.0 has been released, we can remove the 4.8.0 fallback here.
+        CHECKLY_CLI_VERSION: cliVersion ?? '4.8.0',
         CHECKLY_E2E_PROMPTS_INJECTIONS: promptsInjection?.length ? JSON.stringify(promptsInjection) : undefined,
         ...env,
       },

packages/cli/src/commands/test.ts

@@ -8,15 +8,15 @@ import {
   Events,
   RunLocation,
   PrivateRunLocation,
-  CheckRunId,
+  SequenceId,
   DEFAULT_CHECK_RUN_TIMEOUT_SECONDS,
 } from '../services/abstract-check-runner'
 import TestRunner from '../services/test-runner'
 import { loadChecklyConfig } from '../services/checkly-config-loader'
 import { filterByFileNamePattern, filterByCheckNamePattern, filterByTags } from '../services/test-filters'
 import type { Runtime } from '../rest/runtimes'
 import { AuthCommand } from './authCommand'
-import { BrowserCheck, Check, HeartbeatCheck, MultiStepCheck, Project, Session } from '../constructs'
+import { BrowserCheck, Check, HeartbeatCheck, MultiStepCheck, Project, RetryStrategyBuilder, Session } from '../constructs'
 import type { Region } from '..'
 import { splitConfigFilePath, getGitInformation, getCiInformation, getEnvs } from '../services/util'
 import { createReporters, ReporterType } from '../reporters/reporter'
@@ -26,6 +26,7 @@ import { printLn, formatCheckTitle, CheckStatus } from '../reporters/util'
 import { uploadSnapshots } from '../services/snapshot-service'
 
 const DEFAULT_REGION = 'eu-central-1'
+const MAX_RETRIES = 3
 
 export default class Test extends AuthCommand {
   static coreCommand = true
@@ -100,6 +101,9 @@ export default class Test extends AuthCommand {
       description: 'Update any snapshots using the actual result of this test run.',
       default: false,
     }),
+    retries: Flags.integer({
+      description: 'How many times to retry a failing test run.',
+    }),
   }
 
   static args = {
@@ -132,6 +136,7 @@ export default class Test extends AuthCommand {
       record: shouldRecord,
       'test-session-name': testSessionName,
       'update-snapshots': updateSnapshots,
+      retries,
     } = flags
     const filePatterns = argv as string[]
 
@@ -228,6 +233,7 @@ export default class Test extends AuthCommand {
     const reporters = createReporters(reporterTypes, location, verbose)
     const repoInfo = getGitInformation(project.repoUrl)
     const ciInfo = getCiInformation()
+    const testRetryStrategy = this.prepareTestRetryStrategy(retries, checklyConfig?.cli?.retries)
 
     const runner = new TestRunner(
       config.getAccountId(),
@@ -241,34 +247,43 @@ export default class Test extends AuthCommand {
       ciInfo.environment,
       updateSnapshots,
       configDirectory,
+      testRetryStrategy,
     )
 
     runner.on(Events.RUN_STARTED,
-      (checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId: string) =>
+      (checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId: string) =>
         reporters.forEach(r => r.onBegin(checks, testSessionId)),
     )
 
-    runner.on(Events.CHECK_INPROGRESS, (check: any, checkRunId: CheckRunId) => {
-      reporters.forEach(r => r.onCheckInProgress(check, checkRunId))
+    runner.on(Events.CHECK_INPROGRESS, (check: any, sequenceId: SequenceId) => {
+      reporters.forEach(r => r.onCheckInProgress(check, sequenceId))
     })
 
     runner.on(Events.MAX_SCHEDULING_DELAY_EXCEEDED, () => {
       reporters.forEach(r => r.onSchedulingDelayExceeded())
     })
 
-    runner.on(Events.CHECK_SUCCESSFUL, (checkRunId, check, result, links?: TestResultsShortLinks) => {
+    runner.on(Events.CHECK_ATTEMPT_RESULT, (sequenceId: SequenceId, check, result, links?: TestResultsShortLinks) => {
+      reporters.forEach(r => r.onCheckAttemptResult(sequenceId, {
+        logicalId: check.logicalId,
+        sourceFile: check.getSourceFile(),
+        ...result,
+      }, links))
+    })
+
+    runner.on(Events.CHECK_SUCCESSFUL, (sequenceId: SequenceId, check, result, links?: TestResultsShortLinks) => {
       if (result.hasFailures) {
         process.exitCode = 1
       }
 
-      reporters.forEach(r => r.onCheckEnd(checkRunId, {
+      reporters.forEach(r => r.onCheckEnd(sequenceId, {
         logicalId: check.logicalId,
         sourceFile: check.getSourceFile(),
         ...result,
       }, links))
     })
-    runner.on(Events.CHECK_FAILED, (checkRunId, check, message: string) => {
-      reporters.forEach(r => r.onCheckEnd(checkRunId, {
+    runner.on(Events.CHECK_FAILED, (sequenceId: SequenceId, check, message: string) => {
+      reporters.forEach(r => r.onCheckEnd(sequenceId, {
         ...check,
         logicalId: check.logicalId,
         sourceFile: check.getSourceFile(),
@@ -337,6 +352,19 @@ export default class Test extends AuthCommand {
     }
   }
 
+  prepareTestRetryStrategy (retries?: number, configRetries?: number) {
+    const numRetries = retries ?? configRetries ?? 0
+    if (numRetries > MAX_RETRIES) {
+      printLn(`Defaulting to the maximum of ${MAX_RETRIES} retries.`)
+    }
+    return numRetries
+      ? RetryStrategyBuilder.fixedStrategy({
+        maxRetries: Math.min(numRetries, MAX_RETRIES),
+        baseBackoffSeconds: 0,
+      })
+      : null
+  }
+
   private listChecks (checks: Array<Check>) {
     // Sort and print the checks in a way that's consistent with AbstractListReporter
     const sortedCheckFiles = [...new Set(checks.map((check) => check.getSourceFile()))].sort()

packages/cli/src/commands/trigger.ts

@@ -7,13 +7,15 @@ import { loadChecklyConfig } from '../services/checkly-config-loader'
 import { splitConfigFilePath, getEnvs, getGitInformation, getCiInformation } from '../services/util'
 import type { Region } from '..'
 import TriggerRunner, { NoMatchingChecksError } from '../services/trigger-runner'
-import { RunLocation, Events, PrivateRunLocation, CheckRunId } from '../services/abstract-check-runner'
+import { RunLocation, Events, PrivateRunLocation, SequenceId } from '../services/abstract-check-runner'
 import config from '../services/config'
 import { createReporters, ReporterType } from '../reporters/reporter'
+import { printLn } from '../reporters/util'
 import { TestResultsShortLinks } from '../rest/test-sessions'
-import { Session } from '../constructs'
+import { Session, RetryStrategyBuilder } from '../constructs'
 
 const DEFAULT_REGION = 'eu-central-1'
+const MAX_RETRIES = 3
 
 export default class Trigger extends AuthCommand {
   static coreCommand = true
@@ -74,6 +76,9 @@ export default class Trigger extends AuthCommand {
       char: 'n',
       description: 'A name to use when storing results in Checkly with --record.',
     }),
+    retries: Flags.integer({
+      description: 'How many times to retry a check run.',
+    }),
   }
 
   async run (): Promise<void> {
@@ -90,6 +95,7 @@ export default class Trigger extends AuthCommand {
       env,
       'env-file': envFile,
       'test-session-name': testSessionName,
+      retries,
     } = flags
     const envVars = await getEnvs(envFile, env)
     const { configDirectory, configFilenames } = splitConfigFilePath(configFilename)
@@ -106,6 +112,7 @@ export default class Trigger extends AuthCommand {
     const verbose = this.prepareVerboseFlag(verboseFlag, checklyConfig?.cli?.verbose)
     const reporterTypes = this.prepareReportersTypes(reporterFlag as ReporterType, checklyConfig?.cli?.reporters)
     const reporters = createReporters(reporterTypes, location, verbose)
+    const testRetryStrategy = this.prepareTestRetryStrategy(retries, checklyConfig?.cli?.retries)
 
     const repoInfo = getGitInformation()
     const ciInfo = getCiInformation()
@@ -121,19 +128,23 @@ export default class Trigger extends AuthCommand {
       repoInfo,
       ciInfo.environment,
       testSessionName,
+      testRetryStrategy,
     )
     // TODO: This is essentially the same for `checkly test`. Maybe reuse code.
     runner.on(Events.RUN_STARTED,
-      (checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId: string) =>
+      (checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId: string) =>
         reporters.forEach(r => r.onBegin(checks, testSessionId)))
-    runner.on(Events.CHECK_SUCCESSFUL, (checkRunId, _, result, links?: TestResultsShortLinks) => {
+    runner.on(Events.CHECK_ATTEMPT_RESULT, (sequenceId: SequenceId, check, result, links?: TestResultsShortLinks) => {
+      reporters.forEach(r => r.onCheckAttemptResult(sequenceId, result, links))
+    })
+    runner.on(Events.CHECK_SUCCESSFUL, (sequenceId: SequenceId, _, result, links?: TestResultsShortLinks) => {
       if (result.hasFailures) {
         process.exitCode = 1
       }
-      reporters.forEach(r => r.onCheckEnd(checkRunId, result, links))
+      reporters.forEach(r => r.onCheckEnd(sequenceId, result, links))
     })
-    runner.on(Events.CHECK_FAILED, (checkRunId, check, message: string) => {
-      reporters.forEach(r => r.onCheckEnd(checkRunId, {
+    runner.on(Events.CHECK_FAILED, (sequenceId: SequenceId, check, message: string) => {
+      reporters.forEach(r => r.onCheckEnd(sequenceId, {
         ...check,
         hasFailures: true,
         runError: message,
@@ -206,4 +217,17 @@ export default class Trigger extends AuthCommand {
     }
     return reporterFlag ? [reporterFlag] : cliReporters
   }
+
+  prepareTestRetryStrategy (retries?: number, configRetries?: number) {
+    const numRetries = retries ?? configRetries ?? 0
+    if (numRetries > MAX_RETRIES) {
+      printLn(`Defaulting to the maximum of ${MAX_RETRIES} retries.`)
+    }
+    return numRetries
+      ? RetryStrategyBuilder.fixedStrategy({
+        maxRetries: Math.min(numRetries, MAX_RETRIES),
+        baseBackoffSeconds: 0,
+      })
+      : null
+  }
 }

packages/cli/src/reporters/__tests__/__snapshots__/json-builder.spec.ts.snap

@@ -13,7 +13,8 @@ exports[`JsonBuilder renders JSON markdown output with assets & links: json-with
       "durationMilliseconds": 6522,
       "filename": "src/__checks__/folder/browser.check.ts",
       "link": "https://app.checklyhq.com/test-sessions/0c4c64b3-79c5-44a6-ae07-b580ce73f328/results/702961fd-7e2c-45f0-97be-1aa9eabd4d82",
-      "runError": "Run error"
+      "runError": "Run error",
+      "retries": 0
     },
     {
       "result": "Pass",
@@ -22,7 +23,8 @@ exports[`JsonBuilder renders JSON markdown output with assets & links: json-with
       "durationMilliseconds": 1234,
       "filename": "src/some-other-folder/api.check.ts",
       "link": "https://app.checklyhq.com/test-sessions/0c4c64b3-79c5-44a6-ae07-b580ce73f328/results/1c0be612-a5ec-432e-ac1c-837d2f70c010",
-      "runError": "Run error"
+      "runError": "Run error",
+      "retries": 0
     }
   ]
 }"
@@ -40,7 +42,8 @@ exports[`JsonBuilder renders basic JSON output with no assets & links: json-basi
       "durationMilliseconds": 6522,
       "filename": "src/__checks__/folder/browser.check.ts",
       "link": null,
-      "runError": null
+      "runError": null,
+      "retries": 0
     },
     {
       "result": "Pass",
@@ -49,7 +52,8 @@ exports[`JsonBuilder renders basic JSON output with no assets & links: json-basi
       "durationMilliseconds": 1234,
       "filename": "src/some-other-folder/api.check.ts",
       "link": null,
-      "runError": null
+      "runError": null,
+      "retries": 0
     }
   ]
 }"
@@ -67,7 +71,8 @@ exports[`JsonBuilder renders basic JSON output with run errors: json-basic 1`] =
       "durationMilliseconds": 6522,
       "filename": "src/__checks__/folder/browser.check.ts",
       "link": null,
-      "runError": "Run error"
+      "runError": "Run error",
+      "retries": 0
     },
     {
       "result": "Pass",
@@ -76,7 +81,8 @@ exports[`JsonBuilder renders basic JSON output with run errors: json-basic 1`] =
       "durationMilliseconds": 1234,
       "filename": "src/some-other-folder/api.check.ts",
       "link": null,
-      "runError": "Run error"
+      "runError": "Run error",
+      "retries": 0
     }
   ]
 }"

packages/cli/src/reporters/__tests__/fixtures/api-check-result.ts

@@ -5,6 +5,7 @@ export const apiCheckResult = {
   sourceInfo: {
     checkRunId: '4f20dfa7-8c66-4a15-8c43-5dc24f6206c6',
     checkRunSuiteId: '6390a87e-89c7-4295-b6f8-b23e87922ef3',
+    sequenceId: '72c5d10f-fc68-4361-a779-8543575336ae',
     ephemeral: true,
   },
   checkRunId: '1c0be612-a5ec-432e-ac1c-837d2f70c010',

packages/cli/src/reporters/__tests__/fixtures/browser-check-result.ts

@@ -4,6 +4,7 @@ export const browserCheckResult = {
   sourceInfo: {
     checkRunId: 'ebefe140-29bf-4650-9e89-6e36a5c092ef',
     checkRunSuiteId: '2b938869-e38b-4599-b3b0-901aeea4bbef',
+    sequenceId: 'd2ec2faa-a7de-4dc1-8b56-2f4ee9248dc6',
     ephemeral: true,
   },
   checkRunId: '702961fd-7e2c-45f0-97be-1aa9eabd4d82',