feat: check constructs against account default runtime if not given (#…

…1007) The CLI had a hardcoded default runtime value of 2024.02 which became the effective value if no runtime was set at check or project level. Now, the account's default runtime is used instead. Additionally, if the account's default runtime is used, then we leave `runtimeId` undefined when synthesizing resources. This allows the user to have their checks always use the current account default runtime if they wish. If they do not wish to have such behavior, they should set a default runtime at project level as usual.
checkly · Jan 13, 2025 · 611733f · 611733f
1 parent 1fe533f
commit 611733f
Show file tree

Hide file tree

Showing 13 changed files with 263 additions and 18 deletions.
diff --git a/packages/cli/e2e/__tests__/switch.spec.ts b/packages/cli/e2e/__tests__/switch.spec.ts
@@ -11,6 +11,7 @@ describe('switch', () => {
       promptsInjection: [{
         id: config.get('accountId') as string,
         name: accountName,
+        runtimeId: '2024.02', // Not important for this command.
       }],
       timeout: 5000,
     })

diff --git a/packages/cli/src/commands/deploy.ts b/packages/cli/src/commands/deploy.ts
@@ -6,7 +6,6 @@ import { Flags, ux } from '@oclif/core'
 import { AuthCommand } from './authCommand'
 import { parseProject } from '../services/project-parser'
 import { loadChecklyConfig } from '../services/checkly-config-loader'
-import { runtimes } from '../rest/api'
 import type { Runtime } from '../rest/runtimes'
 import {
   Check, AlertChannelSubscription, AlertChannel, CheckGroup, Dashboard,
@@ -92,7 +91,8 @@ export default class Deploy extends AuthCommand {
       config: checklyConfig,
       constructs: checklyConfigConstructs,
     } = await loadChecklyConfig(configDirectory, configFilenames)
-    const { data: avilableRuntimes } = await runtimes.getAll()
+    const { data: account } = await api.accounts.get(config.getAccountId())
+    const { data: avilableRuntimes } = await api.runtimes.getAll()
     const project = await parseProject({
       directory: configDirectory,
       projectLogicalId: checklyConfig.logicalId,
@@ -108,6 +108,7 @@ export default class Deploy extends AuthCommand {
         acc[runtime.name] = runtime
         return acc
       }, <Record<string, Runtime>> {}),
+      defaultRuntimeId: account.runtimeId,
       verifyRuntimeDependencies,
       checklyConfigConstructs,
     })
@@ -140,8 +141,6 @@ export default class Deploy extends AuthCommand {
       return
     }
 
-    const { data: account } = await api.accounts.get(config.getAccountId())
-
     if (!force && !preview) {
       const { confirm } = await prompts({
         name: 'confirm',

diff --git a/packages/cli/src/commands/test.ts b/packages/cli/src/commands/test.ts
@@ -159,6 +159,7 @@ export default class Test extends AuthCommand {
     })
     const verbose = this.prepareVerboseFlag(verboseFlag, checklyConfig.cli?.verbose)
     const reporterTypes = this.prepareReportersTypes(reporterFlag as ReporterType, checklyConfig.cli?.reporters)
+    const { data: account } = await api.accounts.get(config.getAccountId())
     const { data: availableRuntimes } = await api.runtimes.getAll()
 
     const project = await parseProject({
@@ -176,6 +177,7 @@ export default class Test extends AuthCommand {
         acc[runtime.name] = runtime
         return acc
       }, <Record<string, Runtime>> {}),
+      defaultRuntimeId: account.runtimeId,
       verifyRuntimeDependencies,
       checklyConfigConstructs,
     })

diff --git a/packages/cli/src/constructs/__tests__/api-check.spec.ts b/packages/cli/src/constructs/__tests__/api-check.spec.ts
@@ -36,6 +36,84 @@ describe('ApiCheck', () => {
     })
   })
 
+  it('should fail to bundle if runtime is not specified and default runtime is not set', () => {
+    const getFilePath = (filename: string) => path.join(__dirname, 'fixtures', 'api-check', filename)
+    const bundle = () => {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const _bundle = ApiCheck.bundle(getFilePath('entrypoint.js'), undefined)
+    }
+
+    Session.basePath = __dirname
+    Session.availableRuntimes = runtimes
+    Session.defaultRuntimeId = undefined
+    expect(bundle).toThrowError('runtime is not set')
+    delete Session.basePath
+    delete Session.defaultRuntimeId
+  })
+
+  it('should successfully bundle if runtime is not specified but default runtime is set', () => {
+    const getFilePath = (filename: string) => path.join(__dirname, 'fixtures', 'api-check', filename)
+    const bundle = () => {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const _bundle = ApiCheck.bundle(getFilePath('entrypoint.js'), undefined)
+    }
+
+    Session.basePath = __dirname
+    Session.availableRuntimes = runtimes
+    Session.defaultRuntimeId = '2022.10'
+    expect(bundle).not.toThrowError('is not supported')
+    delete Session.basePath
+    delete Session.defaultRuntimeId
+  })
+
+  it('should fail to bundle if runtime is not supported even if default runtime is set', () => {
+    const getFilePath = (filename: string) => path.join(__dirname, 'fixtures', 'api-check', filename)
+    const bundle = () => {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const _bundle = ApiCheck.bundle(getFilePath('entrypoint.js'), '9999.99')
+    }
+
+    Session.basePath = __dirname
+    Session.availableRuntimes = runtimes
+    Session.defaultRuntimeId = '2022.02'
+    expect(bundle).toThrowError('9999.99 is not supported')
+    delete Session.basePath
+    delete Session.defaultRuntimeId
+  })
+
+  it('should not synthesize runtime if not specified even if default runtime is set', () => {
+    Session.project = new Project('project-id', {
+      name: 'Test Project',
+      repoUrl: 'https://github.com/checkly/checkly-cli',
+    })
+    Session.availableRuntimes = runtimes
+    Session.defaultRuntimeId = '2022.02'
+    const apiCheck = new ApiCheck('test-check', {
+      name: 'Test Check',
+      request,
+    })
+    const payload = apiCheck.synthesize()
+    expect(payload.runtimeId).toBeUndefined()
+    delete Session.defaultRuntimeId
+  })
+
+  it('should synthesize runtime if specified', () => {
+    Session.project = new Project('project-id', {
+      name: 'Test Project',
+      repoUrl: 'https://github.com/checkly/checkly-cli',
+    })
+    Session.availableRuntimes = runtimes
+    Session.defaultRuntimeId = '2022.02'
+    const apiCheck = new ApiCheck('test-check', {
+      name: 'Test Check',
+      runtimeId: '2022.02',
+      request,
+    })
+    const payload = apiCheck.synthesize()
+    expect(payload.runtimeId).toEqual('2022.02')
+    delete Session.defaultRuntimeId
+  })
+
   it('should apply default check settings', () => {
     Session.project = new Project('project-id', {
       name: 'Test Project',

diff --git a/packages/cli/src/constructs/__tests__/browser-check.spec.ts b/packages/cli/src/constructs/__tests__/browser-check.spec.ts
@@ -31,6 +31,84 @@ describe('BrowserCheck', () => {
     })
   })
 
+  it('should fail to bundle if runtime is not specified and default runtime is not set', () => {
+    const getFilePath = (filename: string) => path.join(__dirname, 'fixtures', 'api-check', filename)
+    const bundle = () => {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const _bundle = BrowserCheck.bundle(getFilePath('entrypoint.js'), undefined)
+    }
+
+    Session.basePath = __dirname
+    Session.availableRuntimes = runtimes
+    Session.defaultRuntimeId = undefined
+    expect(bundle).toThrowError('runtime is not set')
+    delete Session.basePath
+    delete Session.defaultRuntimeId
+  })
+
+  it('should successfully bundle if runtime is not specified but default runtime is set', () => {
+    const getFilePath = (filename: string) => path.join(__dirname, 'fixtures', 'api-check', filename)
+    const bundle = () => {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const _bundle = BrowserCheck.bundle(getFilePath('entrypoint.js'), undefined)
+    }
+
+    Session.basePath = __dirname
+    Session.availableRuntimes = runtimes
+    Session.defaultRuntimeId = '2022.10'
+    expect(bundle).not.toThrowError('is not supported')
+    delete Session.basePath
+    delete Session.defaultRuntimeId
+  })
+
+  it('should fail to bundle if runtime is not supported even if default runtime is set', () => {
+    const getFilePath = (filename: string) => path.join(__dirname, 'fixtures', 'api-check', filename)
+    const bundle = () => {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const _bundle = BrowserCheck.bundle(getFilePath('entrypoint.js'), '9999.99')
+    }
+
+    Session.basePath = __dirname
+    Session.availableRuntimes = runtimes
+    Session.defaultRuntimeId = '2022.02'
+    expect(bundle).toThrowError('9999.99 is not supported')
+    delete Session.basePath
+    delete Session.defaultRuntimeId
+  })
+
+  it('should not synthesize runtime if not specified even if default runtime is set', () => {
+    Session.project = new Project('project-id', {
+      name: 'Test Project',
+      repoUrl: 'https://github.com/checkly/checkly-cli',
+    })
+    Session.availableRuntimes = runtimes
+    Session.defaultRuntimeId = '2022.02'
+    const browserCheck = new BrowserCheck('test-check', {
+      name: 'Test Check',
+      code: { content: 'console.log("test check")' },
+    })
+    const payload = browserCheck.synthesize()
+    expect(payload.runtimeId).toBeUndefined()
+    delete Session.defaultRuntimeId
+  })
+
+  it('should synthesize runtime if specified', () => {
+    Session.project = new Project('project-id', {
+      name: 'Test Project',
+      repoUrl: 'https://github.com/checkly/checkly-cli',
+    })
+    Session.availableRuntimes = runtimes
+    Session.defaultRuntimeId = '2022.02'
+    const browserCheck = new BrowserCheck('test-check', {
+      name: 'Test Check',
+      runtimeId: '2022.02',
+      code: { content: 'console.log("test check")' },
+    })
+    const payload = browserCheck.synthesize()
+    expect(payload.runtimeId).toEqual('2022.02')
+    delete Session.defaultRuntimeId
+  })
+
   it('should apply default check settings', () => {
     Session.project = new Project('project-id', {
       name: 'Test Project',

diff --git a/packages/cli/src/constructs/__tests__/multi-step-check.spec.ts b/packages/cli/src/constructs/__tests__/multi-step-check.spec.ts
@@ -19,6 +19,7 @@ describe('MultistepCheck', () => {
     })
     expect(check.synthesize()).toMatchObject({ checkType: 'MULTI_STEP' })
   })
+
   it('should throw if runtime does not support multi step check type', () => {
     Session.project = new Project('project-id', {
       name: 'Test Project',
@@ -33,7 +34,75 @@ describe('MultistepCheck', () => {
     }
     expect(() => new MultiStepCheck('main-check', {
       name: 'Main Check',
+      runtimeId: '2023.09',
       code: { content: '' },
     })).toThrowError('This runtime does not support multi step checks.')
   })
+
+  it('should throw if runtime is not set and default runtime does not support multi step check type', () => {
+    Session.project = new Project('project-id', {
+      name: 'Test Project',
+      repoUrl: 'https://github.com/checkly/checkly-cli',
+    })
+    Session.availableRuntimes = {
+      ...runtimesWithMultiStepSupport,
+      9999.99: {
+        ...runtimesWithMultiStepSupport['2023.09'],
+        multiStepSupport: false,
+      },
+    }
+    Session.defaultRuntimeId = '9999.99'
+    expect(() => new MultiStepCheck('main-check', {
+      name: 'Main Check',
+      code: { content: '' },
+    })).toThrowError('This runtime does not support multi step checks.')
+    delete Session.defaultRuntimeId
+  })
+
+  it('should succeed if runtime is not set but default runtime supports multi step check type', () => {
+    Session.project = new Project('project-id', {
+      name: 'Test Project',
+      repoUrl: 'https://github.com/checkly/checkly-cli',
+    })
+    Session.availableRuntimes = runtimesWithMultiStepSupport
+    Session.defaultRuntimeId = '2023.09'
+    expect(() => new MultiStepCheck('main-check', {
+      name: 'Main Check',
+      code: { content: '' },
+    })).not.toThrowError()
+    delete Session.defaultRuntimeId
+  })
+
+  it('should not synthesize runtime if not specified even if default runtime is set', () => {
+    Session.project = new Project('project-id', {
+      name: 'Test Project',
+      repoUrl: 'https://github.com/checkly/checkly-cli',
+    })
+    Session.availableRuntimes = runtimesWithMultiStepSupport
+    Session.defaultRuntimeId = '2023.09'
+    const multiCheck = new MultiStepCheck('main-check', {
+      name: 'Main Check',
+      code: { content: '' },
+    })
+    const payload = multiCheck.synthesize()
+    expect(payload.runtimeId).toBeUndefined()
+    delete Session.defaultRuntimeId
+  })
+
+  it('should synthesize runtime if specified', () => {
+    Session.project = new Project('project-id', {
+      name: 'Test Project',
+      repoUrl: 'https://github.com/checkly/checkly-cli',
+    })
+    Session.availableRuntimes = runtimesWithMultiStepSupport
+    Session.defaultRuntimeId = '2023.09'
+    const multiCheck = new MultiStepCheck('main-check', {
+      name: 'Main Check',
+      runtimeId: '2023.09',
+      code: { content: '' },
+    })
+    const payload = multiCheck.synthesize()
+    expect(payload.runtimeId).toEqual('2023.09')
+    delete Session.defaultRuntimeId
+  })
 })
diff --git a/packages/cli/src/constructs/api-check.ts b/packages/cli/src/constructs/api-check.ts
@@ -295,7 +295,7 @@ export class ApiCheck extends Check {
 
     if (props.setupScript) {
       if ('entrypoint' in props.setupScript) {
-        const { script, scriptPath, dependencies } = ApiCheck.bundle(props.setupScript.entrypoint, this.runtimeId!)
+        const { script, scriptPath, dependencies } = ApiCheck.bundle(props.setupScript.entrypoint, this.runtimeId)
         this.localSetupScript = script
         this.setupScriptPath = scriptPath
         this.setupScriptDependencies = dependencies
@@ -314,7 +314,7 @@ export class ApiCheck extends Check {
 
     if (props.tearDownScript) {
       if ('entrypoint' in props.tearDownScript) {
-        const { script, scriptPath, dependencies } = ApiCheck.bundle(props.tearDownScript.entrypoint, this.runtimeId!)
+        const { script, scriptPath, dependencies } = ApiCheck.bundle(props.tearDownScript.entrypoint, this.runtimeId)
         this.localTearDownScript = script
         this.tearDownScriptPath = scriptPath
         this.tearDownScriptDependencies = dependencies
@@ -337,7 +337,7 @@ export class ApiCheck extends Check {
     this.addPrivateLocationCheckAssignments()
   }
 
-  static bundle (entrypoint: string, runtimeId: string) {
+  static bundle (entrypoint: string, runtimeId?: string) {
     let absoluteEntrypoint = null
     if (path.isAbsolute(entrypoint)) {
       absoluteEntrypoint = entrypoint
@@ -348,7 +348,7 @@ export class ApiCheck extends Check {
       absoluteEntrypoint = path.join(path.dirname(Session.checkFileAbsolutePath), entrypoint)
     }
 
-    const runtime = Session.availableRuntimes[runtimeId]
+    const runtime = Session.getRuntime(runtimeId)
     if (!runtime) {
       throw new Error(`${runtimeId} is not supported`)
     }

diff --git a/packages/cli/src/constructs/browser-check.ts b/packages/cli/src/constructs/browser-check.ts
@@ -79,8 +79,7 @@ export class BrowserCheck extends Check {
         }
         absoluteEntrypoint = path.join(path.dirname(Session.checkFileAbsolutePath), entrypoint)
       }
-      // runtimeId will always be set by check or browser check defaults so it is safe to use ! operator
-      const bundle = BrowserCheck.bundle(absoluteEntrypoint, this.runtimeId!)
+      const bundle = BrowserCheck.bundle(absoluteEntrypoint, this.runtimeId)
       if (!bundle.script) {
         throw new Error(`Browser check "${logicalId}" is not allowed to be empty`)
       }
@@ -115,8 +114,8 @@ export class BrowserCheck extends Check {
     }
   }
 
-  static bundle (entry: string, runtimeId: string) {
-    const runtime = Session.availableRuntimes[runtimeId]
+  static bundle (entry: string, runtimeId?: string) {
+    const runtime = Session.getRuntime(runtimeId)
     if (!runtime) {
       throw new Error(`${runtimeId} is not supported`)
     }

diff --git a/packages/cli/src/constructs/multi-step-check.ts b/packages/cli/src/constructs/multi-step-check.ts
@@ -47,7 +47,7 @@ export class MultiStepCheck extends Check {
 
     this.playwrightConfig = props.playwrightConfig
 
-    if (!Session.availableRuntimes[this.runtimeId!]?.multiStepSupport) {
+    if (!Session.getRuntime(this.runtimeId)?.multiStepSupport) {
       throw new Error('This runtime does not support multi step checks.')
     }
     if ('content' in props.code) {
@@ -65,7 +65,7 @@ export class MultiStepCheck extends Check {
         absoluteEntrypoint = path.join(path.dirname(Session.checkFileAbsolutePath), entrypoint)
       }
       // runtimeId will always be set by check or multi-step check defaults so it is safe to use ! operator
-      const bundle = MultiStepCheck.bundle(absoluteEntrypoint, this.runtimeId!)
+      const bundle = MultiStepCheck.bundle(absoluteEntrypoint, this.runtimeId)
       if (!bundle.script) {
         throw new Error(`Multi-Step check "${logicalId}" is not allowed to be empty`)
       }
@@ -99,8 +99,8 @@ export class MultiStepCheck extends Check {
     }
   }
 
-  static bundle (entry: string, runtimeId: string) {
-    const runtime = Session.availableRuntimes[runtimeId]
+  static bundle (entry: string, runtimeId?: string) {
+    const runtime = Session.getRuntime(runtimeId)
     if (!runtime) {
       throw new Error(`${runtimeId} is not supported`)
     }