feat: add a whole bunch of logging intended to shed light to 408 errors

Although the logging is going to be fairly excessive especially for larger
projects, this should hopefully be enough to give us at least some clue
about the ongoing 408 issues with certain projects.

packages/cli/src/commands/deploy.ts

@@ -17,6 +17,7 @@ import { splitConfigFilePath, getGitInformation } from '../services/util'
 import commonMessages from '../messages/common-messages'
 import { ProjectDeployResponse } from '../rest/projects'
 import { uploadSnapshots } from '../services/snapshot-service'
+import { rootLogger } from '../services/logger'
 
 // eslint-disable-next-line no-restricted-syntax
 enum ResourceDeployStatus {
@@ -58,6 +59,12 @@ export default class Deploy extends AuthCommand {
   }
 
   async run (): Promise<void> {
+    const logger = rootLogger.child({
+      cli: {
+        cmd: 'deploy',
+      },
+    })
+
     ux.action.start('Parsing your project', undefined, { stdout: true })
     const { flags } = await this.parse(Deploy)
     const {
@@ -126,7 +133,21 @@ export default class Deploy extends AuthCommand {
     }
 
     try {
-      const { data } = await api.projects.deploy({ ...projectPayload, repoInfo }, { dryRun: preview, scheduleOnDeploy })
+      const { data } = await api.projects.deploy(
+        { ...projectPayload, repoInfo },
+        { dryRun: preview, scheduleOnDeploy },
+        {
+          onUploadProgress: (event) => {
+            logger.trace({
+              progress: {
+                sent: event.loaded,
+                total: event.total,
+                lengthComputable: event.lengthComputable,
+              },
+            }, 'Project payload upload progress')
+          },
+        },
+      )
       if (preview || output) {
         this.log(this.formatPreview(data, project))
       }

packages/cli/src/commands/test.ts

@@ -24,6 +24,7 @@ import commonMessages from '../messages/common-messages'
 import { TestResultsShortLinks } from '../rest/test-sessions'
 import { printLn, formatCheckTitle, CheckStatus } from '../reporters/util'
 import { uploadSnapshots } from '../services/snapshot-service'
+import { rootLogger } from '../services/logger'
 
 const DEFAULT_REGION = 'eu-central-1'
 const MAX_RETRIES = 3
@@ -118,6 +119,12 @@ export default class Test extends AuthCommand {
   static strict = false
 
   async run (): Promise<void> {
+    const logger = rootLogger.child({
+      cli: {
+        cmd: 'test',
+      },
+    })
+
     ux.action.start('Parsing your project', undefined, { stdout: true })
 
     const { flags, argv } = await this.parse(Test)
@@ -154,6 +161,7 @@ export default class Test extends AuthCommand {
     const reporterTypes = this.prepareReportersTypes(reporterFlag as ReporterType, checklyConfig.cli?.reporters)
     const { data: availableRuntimes } = await api.runtimes.getAll()
 
+    logger.debug('Parsing project')
     const project = await parseProject({
       directory: configDirectory,
       projectLogicalId: checklyConfig.logicalId,
@@ -171,6 +179,8 @@ export default class Test extends AuthCommand {
       }, <Record<string, Runtime>> {}),
       checklyConfigConstructs,
     })
+
+    logger.debug({ project }, 'Finished parsing project')
     const checks = Object.entries(project.data.check)
       .filter(([, check]) => {
         return !(check instanceof HeartbeatCheck)
@@ -211,11 +221,47 @@ export default class Test extends AuthCommand {
         return check
       })
 
+    // Separate browser checks for snapshot uploads.
+    const browserChecks: Array<BrowserCheck> = []
     for (const check of checks) {
       if (!(check instanceof BrowserCheck)) {
         continue
       }
-      check.snapshots = await uploadSnapshots(check.rawSnapshots)
+      if (!check.rawSnapshots?.length) {
+        logger.trace(
+          { check: { logicalId: check.logicalId } },
+          'Check has no snapshots to upload',
+        )
+        continue
+      }
+      browserChecks.push(check)
+    }
+
+    logger.info(
+      { count: browserChecks.length },
+      'Collected all snapshots to upload',
+    )
+
+    for (const check of browserChecks) {
+      const sublogger = logger.child(
+        { check: { logicalId: check.logicalId } },
+      )
+      sublogger.info(
+        { rawSnapshots: check.rawSnapshots },
+        'Uploading check snapshots',
+      )
+      check.snapshots = await uploadSnapshots(check.rawSnapshots, {
+        onUploadProgress: (snapshot, event) => {
+          sublogger.trace({
+            snapshot,
+            progress: {
+              sent: event.loaded,
+              total: event.total,
+              lengthComputable: event.lengthComputable,
+            },
+          }, 'Snapshot upload progress')
+        },
+      })
     }
 
     ux.action.stop()
@@ -236,6 +282,7 @@ export default class Test extends AuthCommand {
     const testRetryStrategy = this.prepareTestRetryStrategy(retries, checklyConfig?.cli?.retries)
 
     const runner = new TestRunner(
+      logger,
       config.getAccountId(),
       project,
       checks,
@@ -251,9 +298,9 @@ export default class Test extends AuthCommand {
     )
 
     runner.on(Events.RUN_STARTED,
-      (checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId: string) =>
-        reporters.forEach(r => r.onBegin(checks, testSessionId)),
-    )
+      (checks: Array<{ check: any, sequenceId: SequenceId }>, testSessionId: string) => {
+        reporters.forEach(r => r.onBegin(checks, testSessionId))
+      })
 
     runner.on(Events.CHECK_INPROGRESS, (check: any, sequenceId: SequenceId) => {
       reporters.forEach(r => r.onCheckInProgress(check, sequenceId))

packages/cli/src/commands/trigger.ts

@@ -19,6 +19,7 @@ import { createReporters, ReporterType } from '../reporters/reporter'
 import { printLn } from '../reporters/util'
 import { TestResultsShortLinks } from '../rest/test-sessions'
 import { Session, RetryStrategyBuilder } from '../constructs'
+import { rootLogger } from '../services/logger'
 
 const DEFAULT_REGION = 'eu-central-1'
 const MAX_RETRIES = 3
@@ -88,6 +89,12 @@ export default class Trigger extends AuthCommand {
   }
 
   async run (): Promise<void> {
+    const logger = rootLogger.child({
+      cli: {
+        cmd: 'trigger',
+      },
+    })
+
     const { flags } = await this.parse(Trigger)
     const {
       location: runLocation,
@@ -124,6 +131,7 @@ export default class Trigger extends AuthCommand {
     const ciInfo = getCiInformation()
 
     const runner = new TriggerRunner(
+      logger,
       config.getAccountId(),
       timeout,
       verbose,

packages/cli/src/rest/checkly-storage.ts

@@ -1,22 +1,36 @@
-import type { AxiosInstance } from 'axios'
+import type { AxiosInstance, AxiosProgressEvent } from 'axios'
 import type { Readable } from 'node:stream'
 
+type UploadOptions = {
+  onUploadProgress?: (progressEvent: AxiosProgressEvent) => void
+}
+
+type DownloadOptions = {
+  onDownloadProgress?: (progressEvent: AxiosProgressEvent) => void
+}
+
 class ChecklyStorage {
   api: AxiosInstance
   constructor (api: AxiosInstance) {
     this.api = api
   }
 
-  upload (stream: Readable) {
+  upload (stream: Readable, options?: UploadOptions) {
     return this.api.post<{ key: string }>(
       '/next/checkly-storage/upload',
       stream,
-      { headers: { 'Content-Type': 'application/octet-stream' } },
+      {
+        headers: { 'Content-Type': 'application/octet-stream' },
+        onUploadProgress: options?.onUploadProgress,
+      },
     )
   }
 
-  download (key: string) {
-    return this.api.post('/next/checkly-storage/download', { key }, { responseType: 'stream' })
+  download (key: string, options?: DownloadOptions) {
+    return this.api.post('/next/checkly-storage/download', { key }, {
+      responseType: 'stream',
+      onDownloadProgress: options?.onDownloadProgress,
+    })
   }
 }
 

packages/cli/src/rest/projects.ts

@@ -1,5 +1,6 @@
-import type { AxiosInstance } from 'axios'
+import type { AxiosInstance, AxiosProgressEvent } from 'axios'
 import type { GitInformation } from '../services/util'
+import Deploy from '../commands/deploy'
 
 export interface Project {
   name: string
@@ -34,6 +35,10 @@ export interface ProjectDeployResponse {
   diff: Array<Change>
 }
 
+export type DeployOptions = {
+  onUploadProgress?: (progressEvent: AxiosProgressEvent) => void
+}
+
 class Projects {
   api: AxiosInstance
   constructor (api: AxiosInstance) {
@@ -56,10 +61,11 @@ class Projects {
     return this.api.delete(`/next/projects/${logicalId}`)
   }
 
-  deploy (resources: ProjectSync, { dryRun = false, scheduleOnDeploy = true } = {}) {
+  deploy (resources: ProjectSync, { dryRun = false, scheduleOnDeploy = true } = {}, options?: DeployOptions) {
     return this.api.post<ProjectDeployResponse>(
       `/next-v2/projects/deploy?dryRun=${dryRun}&scheduleOnDeploy=${scheduleOnDeploy}`,
       resources,
+      { onUploadProgress: options?.onUploadProgress },
     )
   }
 }

packages/cli/src/rest/test-sessions.ts

@@ -1,4 +1,4 @@
-import type { AxiosInstance } from 'axios'
+import type { AxiosInstance, AxiosProgressEvent } from 'axios'
 import { GitInformation } from '../services/util'
 import { RetryStrategy } from '../constructs'
 
@@ -31,14 +31,20 @@ export type TestResultsShortLinks = {
   screenshotLinks: string[]
 }
 
+export type RunOptions = {
+  onUploadProgress?: (progressEvent: AxiosProgressEvent) => void
+}
+
 class TestSessions {
   api: AxiosInstance
   constructor (api: AxiosInstance) {
     this.api = api
   }
 
-  run (payload: RunTestSessionRequest) {
-    return this.api.post('/next/test-sessions/run', payload)
+  run (payload: RunTestSessionRequest, options?: RunOptions) {
+    return this.api.post('/next/test-sessions/run', payload, {
+      onUploadProgress: options?.onUploadProgress,
+    })
   }
 
   trigger (payload: TriggerTestSessionRequest) {