Tnolet/json reporter (#953)

* feat: adds JSON reporter [sc-00]

* docs: fix typo [sc-00]

* feat: removes duration [sc-00]

* fix: handle empty duration [sc-00]

* feat: handle run errors [sc-00]

* fix: commit snapshot [sc-00]

CONTRIBUTING.md

@@ -51,7 +51,7 @@ Make sure you are on the same NodeJS version if you are using `nvm` or `fnm`
 
 ## Running from source in a local folder
 
-You can use the current branch of the code against the any examples in the `/examples` directory for developing and debugging.
+You can use the current branch of the code against any examples in the `/examples` directory for developing and debugging.
 
 1. Go the `~/your_local_path` directory.
 2. Run `npm create checkly -- --template boilerplate-project`

packages/cli/src/commands/test.ts

@@ -81,7 +81,7 @@ export default class Test extends AuthCommand {
     reporter: Flags.string({
       char: 'r',
       description: 'A list of custom reporters for the test output.',
-      options: ['list', 'dot', 'ci', 'github'],
+      options: ['list', 'dot', 'ci', 'github', 'json'],
     }),
     config: Flags.string({
       char: 'c',

packages/cli/src/commands/trigger.ts

@@ -53,7 +53,7 @@ export default class Trigger extends AuthCommand {
     reporter: Flags.string({
       char: 'r',
       description: 'A list of custom reporters for the test output.',
-      options: ['list', 'dot', 'ci', 'github'],
+      options: ['list', 'dot', 'ci', 'github', 'json'],
     }),
     env: Flags.string({
       char: 'e',

packages/cli/src/reporters/__tests__/__snapshots__/json-builder.spec.ts.snap

@@ -0,0 +1,83 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`JsonBuilder renders JSON markdown output with assets & links: json-with-assets-links 1`] = `
+"{
+  "testSessionId": "0c4c64b3-79c5-44a6-ae07-b580ce73f328",
+  "numChecks": 2,
+  "runLocation": "eu-west-1",
+  "checks": [
+    {
+      "result": "Pass",
+      "name": "my-test.spec.ts",
+      "checkType": "BROWSER",
+      "durationMilliseconds": 6522,
+      "filename": "src/__checks__/folder/browser.check.ts",
+      "link": "https://app.checklyhq.com/test-sessions/0c4c64b3-79c5-44a6-ae07-b580ce73f328/results/702961fd-7e2c-45f0-97be-1aa9eabd4d82",
+      "runError": "Run error"
+    },
+    {
+      "result": "Pass",
+      "name": "Test API check",
+      "checkType": "API",
+      "durationMilliseconds": 1234,
+      "filename": "src/some-other-folder/api.check.ts",
+      "link": "https://app.checklyhq.com/test-sessions/0c4c64b3-79c5-44a6-ae07-b580ce73f328/results/1c0be612-a5ec-432e-ac1c-837d2f70c010",
+      "runError": "Run error"
+    }
+  ]
+}"
+`;
+
+exports[`JsonBuilder renders basic JSON output with no assets & links: json-basic 1`] = `
+"{
+  "numChecks": 2,
+  "runLocation": "eu-west-1",
+  "checks": [
+    {
+      "result": "Pass",
+      "name": "my-test.spec.ts",
+      "checkType": "BROWSER",
+      "durationMilliseconds": 6522,
+      "filename": "src/__checks__/folder/browser.check.ts",
+      "link": null,
+      "runError": null
+    },
+    {
+      "result": "Pass",
+      "name": "Test API check",
+      "checkType": "API",
+      "durationMilliseconds": 1234,
+      "filename": "src/some-other-folder/api.check.ts",
+      "link": null,
+      "runError": null
+    }
+  ]
+}"
+`;
+
+exports[`JsonBuilder renders basic JSON output with run errors: json-basic 1`] = `
+"{
+  "numChecks": 2,
+  "runLocation": "eu-west-1",
+  "checks": [
+    {
+      "result": "Pass",
+      "name": "my-test.spec.ts",
+      "checkType": "BROWSER",
+      "durationMilliseconds": 6522,
+      "filename": "src/__checks__/folder/browser.check.ts",
+      "link": null,
+      "runError": "Run error"
+    },
+    {
+      "result": "Pass",
+      "name": "Test API check",
+      "checkType": "API",
+      "durationMilliseconds": 1234,
+      "filename": "src/some-other-folder/api.check.ts",
+      "link": null,
+      "runError": "Run error"
+    }
+  ]
+}"
+`;

packages/cli/src/reporters/__tests__/fixtures/api-check-result.ts

@@ -135,4 +135,5 @@ export const apiCheckResult = {
     requestError: null,
   },
   scheduleError: '',
+  runError: '',
 }

packages/cli/src/reporters/__tests__/fixtures/browser-check-result.ts

@@ -159,4 +159,5 @@ export const browserCheckResult = {
     },
   ],
   scheduleError: '',
+  runError: '',
 }

packages/cli/src/reporters/__tests__/helpers.ts

@@ -2,7 +2,11 @@ import { checkFilesMap } from '../abstract-list'
 import { browserCheckResult } from './fixtures/browser-check-result'
 import { apiCheckResult } from './fixtures/api-check-result'
 
-export function generateMapAndTestResultIds ({ includeTestResultIds = true }) {
+export function generateMapAndTestResultIds ({ includeTestResultIds = true, includeRunErrors = false }) {
+  if (includeRunErrors) {
+    browserCheckResult.runError = 'Run error'
+    apiCheckResult.runError = 'Run error'
+  }
   const checkFilesMapFixture: checkFilesMap = new Map([
     ['folder/browser.check.ts', new Map([
       [browserCheckResult.checkRunId, {

packages/cli/src/reporters/__tests__/json-builder.spec.ts

@@ -0,0 +1,37 @@
+import { JsonBuilder } from '../json'
+import { generateMapAndTestResultIds } from './helpers'
+
+const testSessionId = '0c4c64b3-79c5-44a6-ae07-b580ce73f328'
+const runLocation = 'eu-west-1'
+describe('JsonBuilder', () => {
+  test('renders basic JSON output with no assets & links', () => {
+    const checkFilesMap = generateMapAndTestResultIds({ includeTestResultIds: false })
+    const json = new JsonBuilder({
+      testSessionId: undefined,
+      numChecks: checkFilesMap.size,
+      runLocation,
+      checkFilesMap,
+    }).render()
+    expect(json).toMatchSnapshot('json-basic')
+  })
+  test('renders basic JSON output with run errors', () => {
+    const checkFilesMap = generateMapAndTestResultIds({ includeTestResultIds: false, includeRunErrors: true })
+    const json = new JsonBuilder({
+      testSessionId: undefined,
+      numChecks: checkFilesMap.size,
+      runLocation,
+      checkFilesMap,
+    }).render()
+    expect(json).toMatchSnapshot('json-basic')
+  })
+  test('renders JSON markdown output with assets & links', () => {
+    const checkFilesMap = generateMapAndTestResultIds({ includeTestResultIds: true })
+    const json = new JsonBuilder({
+      testSessionId,
+      numChecks: checkFilesMap.size,
+      runLocation,
+      checkFilesMap,
+    }).render()
+    expect(json).toMatchSnapshot('json-with-assets-links')
+  })
+})

packages/cli/src/reporters/json.ts

@@ -0,0 +1,95 @@
+import * as fs from 'fs'
+import * as path from 'path'
+
+import AbstractListReporter, { checkFilesMap } from './abstract-list'
+import { CheckRunId } from '../services/abstract-check-runner'
+import { printLn, getTestSessionUrl } from './util'
+
+const outputFile = './checkly-json-report.json'
+
+type JsonBuilderOptions = {
+  testSessionId?: string
+  numChecks: number
+  runLocation: string
+  checkFilesMap: checkFilesMap
+}
+
+export class JsonBuilder {
+  testSessionId?: string
+  numChecks: number
+  runLocation: string
+  checkFilesMap: checkFilesMap
+  hasFilenames: boolean
+
+  constructor (options: JsonBuilderOptions) {
+    this.testSessionId = options.testSessionId
+    this.numChecks = options.numChecks
+    this.runLocation = options.runLocation
+    this.checkFilesMap = options.checkFilesMap
+    this.hasFilenames = !(options.checkFilesMap.size === 1 && options.checkFilesMap.has(undefined))
+  }
+
+  render (): string {
+    const testSessionSummary: any = {
+      testSessionId: this.testSessionId,
+      numChecks: this.numChecks,
+      runLocation: this.runLocation,
+      checks: [],
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    for (const [_, checkMap] of this.checkFilesMap.entries()) {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      for (const [_, { result, testResultId }] of checkMap.entries()) {
+        const check: any = {
+          result: result.hasFailures ? 'Fail' : 'Pass',
+          name: result.name,
+          checkType: result.checkType,
+          durationMilliseconds: result.responseTime ?? null,
+          filename: null,
+          link: null,
+          runError: result.runError || null,
+        }
+
+        if (this.hasFilenames) {
+          check.filename = result.sourceFile
+        }
+
+        if (this.testSessionId && testResultId) {
+          check.link = `${getTestSessionUrl(this.testSessionId)}/results/${testResultId}`
+        }
+
+        testSessionSummary.checks.push(check)
+      }
+    }
+
+    return JSON.stringify(testSessionSummary, null, 2)
+  }
+}
+
+export default class JsonReporter extends AbstractListReporter {
+  onBegin (checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId?: string) {
+    super.onBegin(checks, testSessionId)
+    printLn(`Running ${this.numChecks} checks in ${this._runLocationString()}.`, 2, 1)
+  }
+
+  onEnd () {
+    this._printBriefSummary()
+    const jsonBuilder = new JsonBuilder({
+      testSessionId: this.testSessionId,
+      numChecks: this.numChecks!,
+      runLocation: this._runLocationString(),
+      checkFilesMap: this.checkFilesMap!,
+    })
+
+    const json = jsonBuilder.render()
+
+    const summaryFilename = process.env.CHECKLY_REPORTER_JSON_OUTPUT ?? outputFile
+    fs.mkdirSync(path.resolve(path.dirname(summaryFilename)), { recursive: true })
+    fs.writeFileSync(summaryFilename, json)
+
+    printLn(`JSON report saved in '${path.resolve(summaryFilename)}'.`, 2)
+
+    this._printTestSessionsUrl()
+  }
+}

packages/cli/src/reporters/reporter.ts

@@ -4,6 +4,7 @@ import CiReporter from './ci'
 import DotReporter from './dot'
 import GithubReporter from './github'
 import ListReporter from './list'
+import JsonReporter from './json'
 
 export interface Reporter {
   onBegin(checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId?: string): void;
@@ -14,7 +15,7 @@ export interface Reporter {
   onSchedulingDelayExceeded(): void
 }
 
-export type ReporterType = 'list' | 'dot' | 'ci' | 'github'
+export type ReporterType = 'list' | 'dot' | 'ci' | 'github' | 'json'
 
 export const createReporters = (
   types: ReporterType[],
@@ -30,6 +31,8 @@ export const createReporters = (
       return new CiReporter(runLocation, verbose)
     case 'github':
       return new GithubReporter(runLocation, verbose)
+    case 'json':
+      return new JsonReporter(runLocation, verbose)
     default:
       return new ListReporter(runLocation, verbose)
   }