[kbn-scout] add test config category to reporting (elastic#210167)

## Summary This PR adds a test config category to the scout reporting. This allows us to distinguish between UI and API FTR tests. A new property `testConfigCategory` has been added to all FTR configs that don't already inherit it from a higher level config.
joeypoon · Feb 12, 2025 · 4bd8016 · 4bd8016
1 parent a74066d
commit 4bd8016
Show file tree

Hide file tree

Showing 100 changed files with 213 additions and 5 deletions.
diff --git a/packages/kbn-scout-info/src/reporting.ts b/packages/kbn-scout-info/src/reporting.ts
@@ -26,3 +26,11 @@ export const SCOUT_TEST_EVENTS_INDEX_PATTERN =
   process.env.SCOUT_TEST_EVENTS_INDEX_PATTERN || `${SCOUT_TEST_EVENTS_TEMPLATE_NAME}-*`;
 export const SCOUT_TEST_EVENTS_DATA_STREAM_NAME =
   process.env.SCOUT_TEST_EVENTS_DATA_STREAM_NAME || `${SCOUT_TEST_EVENTS_TEMPLATE_NAME}-kibana`;
+
+export enum ScoutTestRunConfigCategory {
+  UI_TEST = 'ui-test',
+  API_TEST = 'api-test',
+  UNIT_TEST = 'unit-test',
+  UNIT_INTEGRATION_TEST = 'unit-integration-test',
+  UNKNOWN = 'unknown',
+}
diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts b/packages/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts
@@ -20,7 +20,7 @@ import type {
 
 import path from 'node:path';
 import { ToolingLog } from '@kbn/tooling-log';
-import { SCOUT_REPORT_OUTPUT_ROOT } from '@kbn/scout-info';
+import { SCOUT_REPORT_OUTPUT_ROOT, ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import stripANSI from 'strip-ansi';
 import { REPO_ROOT } from '@kbn/repo-info';
 import {
@@ -107,6 +107,7 @@ export class ScoutPlaywrightReporter implements Reporter {
     if (config.configFile !== undefined) {
       configInfo = {
         file: this.getScoutFileInfoForPath(path.relative(REPO_ROOT, config.configFile)),
+        category: ScoutTestRunConfigCategory.UI_TEST,
       };
     }
 

diff --git a/packages/kbn-scout-reporting/src/reporting/report/events/event.ts b/packages/kbn-scout-reporting/src/reporting/report/events/event.ts
@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { BuildkiteMetadata, HostMetadata } from '../../../datasources';
 
 /**
@@ -61,6 +62,7 @@ export interface ScoutTestRunInfo {
   id: string;
   config?: {
     file?: ScoutFileInfo;
+    category?: ScoutTestRunConfigCategory;
   };
   status?: string;
   duration?: number;

diff --git a/packages/kbn-scout-reporting/src/reporting/report/events/persistence/mappings.ts b/packages/kbn-scout-reporting/src/reporting/report/events/persistence/mappings.ts
@@ -133,6 +133,9 @@ export const testRunProperties: Record<PropertyName, MappingProperty> = {
         type: 'object',
         properties: fileInfoProperties,
       },
+      category: {
+        type: 'keyword',
+      },
     },
   },
 };

diff --git a/packages/kbn-test/src/functional_test_runner/integration_tests/basic.test.js b/packages/kbn-test/src/functional_test_runner/integration_tests/basic.test.js
@@ -17,7 +17,10 @@ const BASIC_CONFIG = require.resolve('./__fixtures__/simple_project/config.js');
 
 describe('basic config file with a single app and test', function () {
   it('runs and prints expected output', () => {
-    const proc = spawnSync(process.execPath, [SCRIPT, '--config', BASIC_CONFIG]);
+    const proc = spawnSync(process.execPath, [SCRIPT, '--config', BASIC_CONFIG], {
+      // this FTR run should not produce a scout report
+      env: { ...process.env, SCOUT_REPORTER_ENABLED: '0' },
+    });
     const stdout = proc.stdout.toString('utf8');
     expect(stdout).toContain('$BEFORE$');
     expect(stdout).toContain('$TESTNAME$');

diff --git a/packages/kbn-test/src/functional_test_runner/integration_tests/failure_hooks.test.js b/packages/kbn-test/src/functional_test_runner/integration_tests/failure_hooks.test.js
@@ -18,7 +18,10 @@ const FAILURE_HOOKS_CONFIG = require.resolve('./__fixtures__/failure_hooks/confi
 
 describe('failure hooks', function () {
   it('runs and prints expected output', () => {
-    const proc = spawnSync(process.execPath, [SCRIPT, '--config', FAILURE_HOOKS_CONFIG]);
+    const proc = spawnSync(process.execPath, [SCRIPT, '--config', FAILURE_HOOKS_CONFIG], {
+      // this FTR run should not produce a scout report
+      env: { ...process.env, SCOUT_REPORTER_ENABLED: '0' },
+    });
     const lines = stripAnsi(proc.stdout.toString('utf8')).split(/\r?\n/);
     const linesCopy = [...lines];
 

diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts
@@ -11,7 +11,7 @@ import { dirname, resolve } from 'path';
 
 import Joi from 'joi';
 import type { CustomHelpers } from 'joi';
-import { SCOUT_REPORTER_ENABLED } from '@kbn/scout-info';
+import { SCOUT_REPORTER_ENABLED, ScoutTestRunConfigCategory } from '@kbn/scout-info';
 
 // valid pattern for ID
 // enforced camel-case identifiers for consistency
@@ -92,6 +92,9 @@ export const schema = Joi.object()
     testFiles: Joi.array().items(Joi.string()),
     testRunner: Joi.func(),
     serverless: Joi.boolean().default(false),
+    testConfigCategory: Joi.string()
+      .valid(...Object.values(ScoutTestRunConfigCategory))
+      .default(ScoutTestRunConfigCategory.UNKNOWN),
 
     suiteFiles: Joi.object()
       .keys({

diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/scout_ftr_reporter.ts b/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/scout_ftr_reporter.ts
@@ -68,7 +68,10 @@ export class ScoutFTRReporter {
     this.codeOwnersEntries = getCodeOwnersEntries();
     this.baseTestRunInfo = {
       id: this.runId,
-      config: { file: this.getScoutFileInfoForPath(path.relative(REPO_ROOT, config.path)) },
+      config: {
+        file: this.getScoutFileInfoForPath(path.relative(REPO_ROOT, config.path)),
+        category: config.get('testConfigCategory'),
+      },
     };
 
     // Register event listeners

diff --git a/test/analytics/config.ts b/test/analytics/config.ts
@@ -9,6 +9,7 @@
 
 import path from 'path';
 import { FtrConfigProviderContext } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services } from './services';
 
 /*
@@ -23,6 +24,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('./tests')],
     services,
     pageObjects: functionalConfig.get('pageObjects'),

diff --git a/test/api_integration/config.js b/test/api_integration/config.js
@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services } from './services';
 
 export default async function ({ readConfigFile }) {
@@ -18,6 +19,7 @@ export default async function ({ readConfigFile }) {
     testFiles: [require.resolve('./apis')],
     services,
     servers: commonConfig.get('servers'),
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     junit: {
       reportName: 'API Integration Tests',
     },

diff --git a/test/examples/config.js b/test/examples/config.js
@@ -10,12 +10,14 @@
 import { resolve } from 'path';
 import { REPO_ROOT } from '@kbn/repo-info';
 import { findTestPluginPaths } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { services } from '../plugin_functional/services';
 
 export default async function ({ readConfigFile }) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     rootTags: ['runOutsideOfCiGroups'],
     testFiles: [
       require.resolve('./hello_world'),

diff --git a/test/functional/config.base.js b/test/functional/config.base.js
@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { pageObjects } from './page_objects';
 import { services } from './services';
 
@@ -19,6 +20,8 @@ export default async function ({ readConfigFile }) {
 
     servers: commonConfig.get('servers'),
 
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
+
     esTestCluster: {
       ...commonConfig.get('esTestCluster'),
       serverArgs: [`xpack.security.enabled=${process.env.ES_SECURITY_ENABLED ? 'true' : 'false'}`],

diff --git a/test/health_gateway/config.ts b/test/health_gateway/config.ts
@@ -8,13 +8,15 @@
  */
 
 import path from 'path';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { services } from './services';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     services,
     rootTags: ['runOutsideOfCiGroups'],
     esTestCluster: functionalConfig.get('esTestCluster'),

diff --git a/test/interactive_setup_api_integration/manual_configuration_flow_without_tls.config.ts b/test/interactive_setup_api_integration/manual_configuration_flow_without_tls.config.ts
@@ -10,6 +10,7 @@
 import fs from 'fs/promises';
 import { join, resolve } from 'path';
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 import { getDataPath } from '@kbn/utils';
 
@@ -24,6 +25,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   await fs.writeFile(tempKibanaYamlFile, '');
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
     testFiles: [require.resolve('./tests/manual_configuration_flow_without_tls')],
     servers: xPackAPITestsConfig.get('servers'),
     services,

diff --git a/test/interpreter_functional/config.ts b/test/interpreter_functional/config.ts
@@ -8,12 +8,14 @@
  */
 
 import path from 'path';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext, findTestPluginPaths } from '@kbn/test';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     rootTags: ['runOutsideOfCiGroups'],
     testFiles: [require.resolve('./test_suites/run_pipeline')],
     services: functionalConfig.get('services'),

diff --git a/test/node_roles_functional/all.config.ts b/test/node_roles_functional/all.config.ts
@@ -7,13 +7,15 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import path from 'path';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     rootTags: ['runOutsideOfCiGroups'],
     testFiles: [require.resolve('./test_suites/all')],
     services: {

diff --git a/test/node_roles_functional/background_tasks.config.ts b/test/node_roles_functional/background_tasks.config.ts
@@ -7,13 +7,15 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import path from 'path';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     rootTags: ['runOutsideOfCiGroups'],
     testFiles: [require.resolve('./test_suites/background_tasks')],
     services: {

diff --git a/test/node_roles_functional/ui.config.ts b/test/node_roles_functional/ui.config.ts
@@ -7,13 +7,15 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 import path from 'path';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     rootTags: ['runOutsideOfCiGroups'],
     testFiles: [require.resolve('./test_suites/ui')],
     services: {

diff --git a/test/plugin_functional/config.ts b/test/plugin_functional/config.ts
@@ -8,12 +8,14 @@
  */
 
 import { FtrConfigProviderContext, findTestPluginPaths } from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import path from 'path';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     rootTags: ['runOutsideOfCiGroups'],
     testFiles: [
       require.resolve('./test_suites/usage_collection'),

diff --git a/test/server_integration/http/platform/config.status.ts b/test/server_integration/http/platform/config.status.ts
@@ -8,6 +8,7 @@
  */
 
 import path from 'path';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext, findTestPluginPaths } from '@kbn/test';
 
 /*
@@ -22,6 +23,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const httpConfig = await readConfigFile(require.resolve('../../config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [
       // Status test should be first to resolve manually created "unavailable" plugin
       require.resolve('./status'),

diff --git a/test/server_integration/http/platform/config.ts b/test/server_integration/http/platform/config.ts
@@ -7,12 +7,14 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const httpConfig = await readConfigFile(require.resolve('../../config.base.js'));
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('./cache'), require.resolve('./headers')],
     services: httpConfig.get('services'),
     servers: httpConfig.get('servers'),

diff --git a/test/server_integration/http/ssl/config.js b/test/server_integration/http/ssl/config.js
@@ -9,13 +9,15 @@
 
 import { readFileSync } from 'fs';
 import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { createKibanaSupertestProvider } from '../../services';
 
 export default async function ({ readConfigFile }) {
   const httpConfig = await readConfigFile(require.resolve('../../config.base.js'));
   const certificateAuthorities = [readFileSync(CA_CERT_PATH)];
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('.')],
     services: {
       ...httpConfig.get('services'),

diff --git a/test/server_integration/http/ssl_redirect/config.ts b/test/server_integration/http/ssl_redirect/config.ts
@@ -10,6 +10,7 @@
 import Url from 'url';
 import { readFileSync } from 'fs';
 import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 
 import { createKibanaSupertestProvider } from '../../services';
@@ -21,6 +22,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const redirectPort = httpConfig.get('servers.kibana.port') + 1234;
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('.')],
     services: {
       ...httpConfig.get('services'),

diff --git a/test/server_integration/http/ssl_with_p12/config.js b/test/server_integration/http/ssl_with_p12/config.js
@@ -9,13 +9,15 @@
 
 import { readFileSync } from 'fs';
 import { CA_CERT_PATH, KBN_P12_PATH, KBN_P12_PASSWORD } from '@kbn/dev-utils';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { createKibanaSupertestProvider } from '../../services';
 
 export default async function ({ readConfigFile }) {
   const httpConfig = await readConfigFile(require.resolve('../../config.base.js'));
   const certificateAuthorities = [readFileSync(CA_CERT_PATH)];
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('.')],
     services: {
       ...httpConfig.get('services'),

diff --git a/test/server_integration/http/ssl_with_p12_intermediate/config.js b/test/server_integration/http/ssl_with_p12_intermediate/config.js
@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import { readFileSync } from 'fs';
 import { CA1_CERT_PATH, CA2_CERT_PATH, EE_P12_PATH, EE_P12_PASSWORD } from '../../__fixtures__';
 import { createKibanaSupertestProvider } from '../../services';
@@ -16,6 +17,7 @@ export default async function ({ readConfigFile }) {
   const certificateAuthorities = [readFileSync(CA1_CERT_PATH), readFileSync(CA2_CERT_PATH)];
 
   return {
+    testConfigCategory: ScoutTestRunConfigCategory.UI_TEST,
     testFiles: [require.resolve('.')],
     services: {
       ...httpConfig.get('services'),

diff --git a/test/tsconfig.json b/test/tsconfig.json
@@ -77,5 +77,6 @@
     "@kbn/core-saved-objects-import-export-server-internal",
     "@kbn/core-deprecations-common",
     "@kbn/data-grid-in-table-search",
+    "@kbn/scout-info",
   ]
 }