diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bac2c340..fc4355c5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,7 +51,7 @@ Make sure you are on the same NodeJS version if you are using `nvm` or `fnm` ## Running from source in a local folder -You can use the current branch of the code against the any examples in the `/examples` directory for developing and debugging. +You can use the current branch of the code against any examples in the `/examples` directory for developing and debugging. 1. Go the `~/your_local_path` directory. 2. Run `npm create checkly -- --template boilerplate-project` diff --git a/packages/cli/src/commands/test.ts b/packages/cli/src/commands/test.ts index 38699446..9806ee62 100644 --- a/packages/cli/src/commands/test.ts +++ b/packages/cli/src/commands/test.ts @@ -81,7 +81,7 @@ export default class Test extends AuthCommand { reporter: Flags.string({ char: 'r', description: 'A list of custom reporters for the test output.', - options: ['list', 'dot', 'ci', 'github'], + options: ['list', 'dot', 'ci', 'github', 'json'], }), config: Flags.string({ char: 'c', diff --git a/packages/cli/src/commands/trigger.ts b/packages/cli/src/commands/trigger.ts index b2be57fc..c7cda64f 100644 --- a/packages/cli/src/commands/trigger.ts +++ b/packages/cli/src/commands/trigger.ts @@ -53,7 +53,7 @@ export default class Trigger extends AuthCommand { reporter: Flags.string({ char: 'r', description: 'A list of custom reporters for the test output.', - options: ['list', 'dot', 'ci', 'github'], + options: ['list', 'dot', 'ci', 'github', 'json'], }), env: Flags.string({ char: 'e', diff --git a/packages/cli/src/reporters/__tests__/__snapshots__/json-builder.spec.ts.snap b/packages/cli/src/reporters/__tests__/__snapshots__/json-builder.spec.ts.snap new file mode 100644 index 00000000..e163922b --- /dev/null +++ b/packages/cli/src/reporters/__tests__/__snapshots__/json-builder.spec.ts.snap @@ -0,0 +1,83 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`JsonBuilder renders JSON markdown output with assets & links: json-with-assets-links 1`] = ` +"{ + "testSessionId": "0c4c64b3-79c5-44a6-ae07-b580ce73f328", + "numChecks": 2, + "runLocation": "eu-west-1", + "checks": [ + { + "result": "Pass", + "name": "my-test.spec.ts", + "checkType": "BROWSER", + "durationMilliseconds": 6522, + "filename": "src/__checks__/folder/browser.check.ts", + "link": "https://app.checklyhq.com/test-sessions/0c4c64b3-79c5-44a6-ae07-b580ce73f328/results/702961fd-7e2c-45f0-97be-1aa9eabd4d82", + "runError": "Run error" + }, + { + "result": "Pass", + "name": "Test API check", + "checkType": "API", + "durationMilliseconds": 1234, + "filename": "src/some-other-folder/api.check.ts", + "link": "https://app.checklyhq.com/test-sessions/0c4c64b3-79c5-44a6-ae07-b580ce73f328/results/1c0be612-a5ec-432e-ac1c-837d2f70c010", + "runError": "Run error" + } + ] +}" +`; + +exports[`JsonBuilder renders basic JSON output with no assets & links: json-basic 1`] = ` +"{ + "numChecks": 2, + "runLocation": "eu-west-1", + "checks": [ + { + "result": "Pass", + "name": "my-test.spec.ts", + "checkType": "BROWSER", + "durationMilliseconds": 6522, + "filename": "src/__checks__/folder/browser.check.ts", + "link": null, + "runError": null + }, + { + "result": "Pass", + "name": "Test API check", + "checkType": "API", + "durationMilliseconds": 1234, + "filename": "src/some-other-folder/api.check.ts", + "link": null, + "runError": null + } + ] +}" +`; + +exports[`JsonBuilder renders basic JSON output with run errors: json-basic 1`] = ` +"{ + "numChecks": 2, + "runLocation": "eu-west-1", + "checks": [ + { + "result": "Pass", + "name": "my-test.spec.ts", + "checkType": "BROWSER", + "durationMilliseconds": 6522, + "filename": "src/__checks__/folder/browser.check.ts", + "link": null, + "runError": "Run error" + }, + { + "result": "Pass", + "name": "Test API check", + "checkType": "API", + "durationMilliseconds": 1234, + "filename": "src/some-other-folder/api.check.ts", + "link": null, + "runError": "Run error" + } + ] +}" +`; diff --git a/packages/cli/src/reporters/__tests__/fixtures/api-check-result.ts b/packages/cli/src/reporters/__tests__/fixtures/api-check-result.ts index 83f28c18..eb98ad30 100644 --- a/packages/cli/src/reporters/__tests__/fixtures/api-check-result.ts +++ b/packages/cli/src/reporters/__tests__/fixtures/api-check-result.ts @@ -135,4 +135,5 @@ export const apiCheckResult = { requestError: null, }, scheduleError: '', + runError: '', } diff --git a/packages/cli/src/reporters/__tests__/fixtures/browser-check-result.ts b/packages/cli/src/reporters/__tests__/fixtures/browser-check-result.ts index a760b8d8..e559374f 100644 --- a/packages/cli/src/reporters/__tests__/fixtures/browser-check-result.ts +++ b/packages/cli/src/reporters/__tests__/fixtures/browser-check-result.ts @@ -159,4 +159,5 @@ export const browserCheckResult = { }, ], scheduleError: '', + runError: '', } diff --git a/packages/cli/src/reporters/__tests__/helpers.ts b/packages/cli/src/reporters/__tests__/helpers.ts index 1f2672c2..cb01d704 100644 --- a/packages/cli/src/reporters/__tests__/helpers.ts +++ b/packages/cli/src/reporters/__tests__/helpers.ts @@ -2,7 +2,11 @@ import { checkFilesMap } from '../abstract-list' import { browserCheckResult } from './fixtures/browser-check-result' import { apiCheckResult } from './fixtures/api-check-result' -export function generateMapAndTestResultIds ({ includeTestResultIds = true }) { +export function generateMapAndTestResultIds ({ includeTestResultIds = true, includeRunErrors = false }) { + if (includeRunErrors) { + browserCheckResult.runError = 'Run error' + apiCheckResult.runError = 'Run error' + } const checkFilesMapFixture: checkFilesMap = new Map([ ['folder/browser.check.ts', new Map([ [browserCheckResult.checkRunId, { diff --git a/packages/cli/src/reporters/__tests__/json-builder.spec.ts b/packages/cli/src/reporters/__tests__/json-builder.spec.ts new file mode 100644 index 00000000..58e58329 --- /dev/null +++ b/packages/cli/src/reporters/__tests__/json-builder.spec.ts @@ -0,0 +1,37 @@ +import { JsonBuilder } from '../json' +import { generateMapAndTestResultIds } from './helpers' + +const testSessionId = '0c4c64b3-79c5-44a6-ae07-b580ce73f328' +const runLocation = 'eu-west-1' +describe('JsonBuilder', () => { + test('renders basic JSON output with no assets & links', () => { + const checkFilesMap = generateMapAndTestResultIds({ includeTestResultIds: false }) + const json = new JsonBuilder({ + testSessionId: undefined, + numChecks: checkFilesMap.size, + runLocation, + checkFilesMap, + }).render() + expect(json).toMatchSnapshot('json-basic') + }) + test('renders basic JSON output with run errors', () => { + const checkFilesMap = generateMapAndTestResultIds({ includeTestResultIds: false, includeRunErrors: true }) + const json = new JsonBuilder({ + testSessionId: undefined, + numChecks: checkFilesMap.size, + runLocation, + checkFilesMap, + }).render() + expect(json).toMatchSnapshot('json-basic') + }) + test('renders JSON markdown output with assets & links', () => { + const checkFilesMap = generateMapAndTestResultIds({ includeTestResultIds: true }) + const json = new JsonBuilder({ + testSessionId, + numChecks: checkFilesMap.size, + runLocation, + checkFilesMap, + }).render() + expect(json).toMatchSnapshot('json-with-assets-links') + }) +}) diff --git a/packages/cli/src/reporters/json.ts b/packages/cli/src/reporters/json.ts new file mode 100644 index 00000000..5c9dae91 --- /dev/null +++ b/packages/cli/src/reporters/json.ts @@ -0,0 +1,95 @@ +import * as fs from 'fs' +import * as path from 'path' + +import AbstractListReporter, { checkFilesMap } from './abstract-list' +import { CheckRunId } from '../services/abstract-check-runner' +import { printLn, getTestSessionUrl } from './util' + +const outputFile = './checkly-json-report.json' + +type JsonBuilderOptions = { + testSessionId?: string + numChecks: number + runLocation: string + checkFilesMap: checkFilesMap +} + +export class JsonBuilder { + testSessionId?: string + numChecks: number + runLocation: string + checkFilesMap: checkFilesMap + hasFilenames: boolean + + constructor (options: JsonBuilderOptions) { + this.testSessionId = options.testSessionId + this.numChecks = options.numChecks + this.runLocation = options.runLocation + this.checkFilesMap = options.checkFilesMap + this.hasFilenames = !(options.checkFilesMap.size === 1 && options.checkFilesMap.has(undefined)) + } + + render (): string { + const testSessionSummary: any = { + testSessionId: this.testSessionId, + numChecks: this.numChecks, + runLocation: this.runLocation, + checks: [], + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for (const [_, checkMap] of this.checkFilesMap.entries()) { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for (const [_, { result, testResultId }] of checkMap.entries()) { + const check: any = { + result: result.hasFailures ? 'Fail' : 'Pass', + name: result.name, + checkType: result.checkType, + durationMilliseconds: result.responseTime ?? null, + filename: null, + link: null, + runError: result.runError || null, + } + + if (this.hasFilenames) { + check.filename = result.sourceFile + } + + if (this.testSessionId && testResultId) { + check.link = `${getTestSessionUrl(this.testSessionId)}/results/${testResultId}` + } + + testSessionSummary.checks.push(check) + } + } + + return JSON.stringify(testSessionSummary, null, 2) + } +} + +export default class JsonReporter extends AbstractListReporter { + onBegin (checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId?: string) { + super.onBegin(checks, testSessionId) + printLn(`Running ${this.numChecks} checks in ${this._runLocationString()}.`, 2, 1) + } + + onEnd () { + this._printBriefSummary() + const jsonBuilder = new JsonBuilder({ + testSessionId: this.testSessionId, + numChecks: this.numChecks!, + runLocation: this._runLocationString(), + checkFilesMap: this.checkFilesMap!, + }) + + const json = jsonBuilder.render() + + const summaryFilename = process.env.CHECKLY_REPORTER_JSON_OUTPUT ?? outputFile + fs.mkdirSync(path.resolve(path.dirname(summaryFilename)), { recursive: true }) + fs.writeFileSync(summaryFilename, json) + + printLn(`JSON report saved in '${path.resolve(summaryFilename)}'.`, 2) + + this._printTestSessionsUrl() + } +} diff --git a/packages/cli/src/reporters/reporter.ts b/packages/cli/src/reporters/reporter.ts index 759934d5..6e622390 100644 --- a/packages/cli/src/reporters/reporter.ts +++ b/packages/cli/src/reporters/reporter.ts @@ -4,6 +4,7 @@ import CiReporter from './ci' import DotReporter from './dot' import GithubReporter from './github' import ListReporter from './list' +import JsonReporter from './json' export interface Reporter { onBegin(checks: Array<{ check: any, checkRunId: CheckRunId, testResultId?: string }>, testSessionId?: string): void; @@ -14,7 +15,7 @@ export interface Reporter { onSchedulingDelayExceeded(): void } -export type ReporterType = 'list' | 'dot' | 'ci' | 'github' +export type ReporterType = 'list' | 'dot' | 'ci' | 'github' | 'json' export const createReporters = ( types: ReporterType[], @@ -30,6 +31,8 @@ export const createReporters = ( return new CiReporter(runLocation, verbose) case 'github': return new GithubReporter(runLocation, verbose) + case 'json': + return new JsonReporter(runLocation, verbose) default: return new ListReporter(runLocation, verbose) }