diff --git a/cmd/testrunner.go b/cmd/testrunner.go index 284b927a5..bf9c3150e 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -25,7 +25,11 @@ import ( "github.com/elastic/elastic-package/internal/testrunner" "github.com/elastic/elastic-package/internal/testrunner/reporters/formats" "github.com/elastic/elastic-package/internal/testrunner/reporters/outputs" - _ "github.com/elastic/elastic-package/internal/testrunner/runners" // register all test runners + "github.com/elastic/elastic-package/internal/testrunner/runners/asset" + "github.com/elastic/elastic-package/internal/testrunner/runners/pipeline" + "github.com/elastic/elastic-package/internal/testrunner/runners/policy" + "github.com/elastic/elastic-package/internal/testrunner/runners/static" + "github.com/elastic/elastic-package/internal/testrunner/runners/system" ) const testLongDescription = `Use this command to run tests on a package. Currently, the following types of tests are available: @@ -164,16 +168,13 @@ func testRunnerAssetCommandAction(cmd *cobra.Command, args []string) error { _, pkg := filepath.Split(packageRootPath) - results, err := testrunner.Run(ctx, testType, testrunner.TestOptions{ - Profile: profile, - TestFolder: testrunner.TestFolder{Package: pkg}, - PackageRootPath: packageRootPath, - WithCoverage: testCoverage, - CoverageType: testCoverageFormat, - KibanaClient: kibanaClient, - RunIndependentElasticAgent: false, + runner := asset.NewAssetRunner(asset.AssetRunnerOptions{ + TestFolder: testrunner.TestFolder{Package: pkg}, + PackageRootPath: packageRootPath, + KibanaClient: kibanaClient, }) + results, err := testrunner.Run(ctx, runner) if err != nil { return fmt.Errorf("error running package %s tests: %w", testType, err) } @@ -200,11 +201,6 @@ func testRunnerStaticCommandAction(cmd *cobra.Command, args []string) error { cmd.Printf("Run static tests for the package\n") testType := testrunner.TestType("static") - profile, err := cobraext.GetProfileFlag(cmd) - if err != nil { - return err - } - failOnMissing, err := cmd.Flags().GetBool(cobraext.FailOnMissingFlagName) if err != nil { return cobraext.FlagParsingError(err, cobraext.FailOnMissingFlagName) @@ -295,20 +291,15 @@ func testRunnerStaticCommandAction(cmd *cobra.Command, args []string) error { var results []testrunner.TestResult for _, folder := range testFolders { - r, err := testrunner.Run(ctx, testType, testrunner.TestOptions{ - Profile: profile, - TestFolder: folder, - PackageRootPath: packageRootPath, - RunIndependentElasticAgent: false, + runner := static.NewStaticRunner(static.StaticRunnerOptions{ + TestFolder: folder, + PackageRootPath: packageRootPath, }) - - // Results must be appended even if there is an error, since there could be - // tests (e.g. system tests) that return both error and results. - results = append(results, r...) - + r, err := testrunner.Run(ctx, runner) if err != nil { return fmt.Errorf("error running package %s tests: %w", testType, err) } + results = append(results, r...) } return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage) @@ -449,25 +440,25 @@ func testRunnerPipelineCommandAction(cmd *cobra.Command, args []string) error { var results []testrunner.TestResult for _, folder := range testFolders { - r, err := testrunner.Run(ctx, testType, testrunner.TestOptions{ - Profile: profile, - TestFolder: folder, - PackageRootPath: packageRootPath, - GenerateTestResult: generateTestResult, - API: esClient.API, - WithCoverage: testCoverage, - CoverageType: testCoverageFormat, - DeferCleanup: deferCleanup, - RunIndependentElasticAgent: false, + runner, err := pipeline.NewPipelineRunner(pipeline.PipelineRunnerOptions{ + Profile: profile, + TestFolder: folder, + PackageRootPath: packageRootPath, + GenerateTestResult: generateTestResult, + API: esClient.API, + WithCoverage: testCoverage, + CoverageType: testCoverageFormat, + DeferCleanup: deferCleanup, }) + if err != nil { + return fmt.Errorf("failed to create pipeline runner: %w", err) + } - // Results must be appended even if there is an error, since there could be - // tests (e.g. system tests) that return both error and results. - results = append(results, r...) - + r, err := testrunner.Run(ctx, runner) if err != nil { return fmt.Errorf("error running package %s tests: %w", testType, err) } + results = append(results, r...) } return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage) @@ -691,7 +682,7 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error { var results []testrunner.TestResult for _, folder := range testFolders { - r, err := testrunner.Run(ctx, testType, testrunner.TestOptions{ + runner := system.NewSystemRunner(system.SystemRunnerOptions{ Profile: profile, TestFolder: folder, PackageRootPath: packageRootPath, @@ -707,13 +698,11 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error { RunIndependentElasticAgent: false, }) - // Results must be appended even if there is an error, since there could be - // tests (e.g. system tests) that return both error and results. - results = append(results, r...) - + r, err := testrunner.Run(ctx, runner) if err != nil { return fmt.Errorf("error running package %s tests: %w", testType, err) } + results = append(results, r...) } return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage) @@ -845,22 +834,17 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error { var results []testrunner.TestResult for _, folder := range testFolders { - r, err := testrunner.Run(ctx, testType, testrunner.TestOptions{ - Profile: profile, - TestFolder: folder, - PackageRootPath: packageRootPath, - GenerateTestResult: generateTestResult, - KibanaClient: kibanaClient, - RunIndependentElasticAgent: false, + runner := policy.NewPolicyRunner(policy.PolicyRunnerOptions{ + TestFolder: folder, + PackageRootPath: packageRootPath, + GenerateTestResult: generateTestResult, + KibanaClient: kibanaClient, }) - - // Results must be appended even if there is an error, since there could be - // tests (e.g. system tests) that return both error and results. - results = append(results, r...) - + r, err := testrunner.Run(ctx, runner) if err != nil { return fmt.Errorf("error running package %s tests: %w", testType, err) } + results = append(results, r...) } return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage) diff --git a/internal/testrunner/runners/asset/runner.go b/internal/testrunner/runners/asset/runner.go index 963f2ca49..e87fad086 100644 --- a/internal/testrunner/runners/asset/runner.go +++ b/internal/testrunner/runners/asset/runner.go @@ -17,10 +17,6 @@ import ( "github.com/elastic/elastic-package/internal/testrunner" ) -func init() { - testrunner.RegisterRunner(&runner{}) -} - const ( // TestType defining asset loading tests TestType testrunner.TestType = "asset" @@ -33,6 +29,26 @@ type runner struct { resourcesManager *resources.Manager } +type AssetRunnerOptions struct { + TestFolder testrunner.TestFolder + PackageRootPath string + KibanaClient *kibana.Client +} + +func NewAssetRunner(options AssetRunnerOptions) *runner { + runner := runner{ + testFolder: options.TestFolder, + packageRootPath: options.PackageRootPath, + kibanaClient: options.KibanaClient, + } + + manager := resources.NewManager() + manager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: options.KibanaClient}) + runner.resourcesManager = manager + + return &runner +} + // Ensures that runner implements testrunner.TestRunner interface var _ testrunner.TestRunner = new(runner) @@ -47,15 +63,7 @@ func (r runner) String() string { } // Run runs the asset loading tests -func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]testrunner.TestResult, error) { - r.testFolder = options.TestFolder - r.packageRootPath = options.PackageRootPath - r.kibanaClient = options.KibanaClient - - manager := resources.NewManager() - manager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: r.kibanaClient}) - r.resourcesManager = manager - +func (r *runner) Run(ctx context.Context) ([]testrunner.TestResult, error) { return r.run(ctx) } diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index 62b5fa44f..e36e9a84f 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -17,8 +17,8 @@ import ( "github.com/elastic/elastic-package/internal/testrunner" ) -// GetPipelineCoverage returns a coverage report for the provided set of ingest pipelines. -func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipeline) (testrunner.CoverageReport, error) { +// getPipelineCoverage returns a coverage report for the provided set of ingest pipelines. +func getPipelineCoverage(options PipelineRunnerOptions, pipelines []ingest.Pipeline) (testrunner.CoverageReport, error) { dataStreamPath, found, err := packages.FindDataStreamRootForPath(options.TestFolder.Path) if err != nil { return nil, fmt.Errorf("locating data_stream root failed: %w", err) diff --git a/internal/testrunner/runners/pipeline/runner.go b/internal/testrunner/runners/pipeline/runner.go index 36918dae1..2f8b10628 100644 --- a/internal/testrunner/runners/pipeline/runner.go +++ b/internal/testrunner/runners/pipeline/runner.go @@ -21,6 +21,7 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/elastic-package/internal/common" + "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/elasticsearch/ingest" "github.com/elastic/elastic-package/internal/environment" "github.com/elastic/elastic-package/internal/fields" @@ -28,6 +29,7 @@ import ( "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/multierror" "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/stack" "github.com/elastic/elastic-package/internal/testrunner" ) @@ -40,7 +42,15 @@ const ( var serverlessDisableCompareResults = environment.WithElasticPackagePrefix("SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS") type runner struct { - options testrunner.TestOptions + profile *profile.Profile + deferCleanup time.Duration + esAPI *elasticsearch.API + packageRootPath string + testFolder testrunner.TestFolder + generateTestResult bool + withCoverage bool + coverageType string + pipelines []ingest.Pipeline runCompareResults bool @@ -48,35 +58,35 @@ type runner struct { provider stack.Provider } -type IngestPipelineReroute struct { - Description string `yaml:"description"` - Processors []map[string]ingest.RerouteProcessor `yaml:"processors"` - AdditionalFields map[string]interface{} `yaml:",inline"` -} - -// Ensures that runner implements testrunner.TestRunner interface -var _ testrunner.TestRunner = new(runner) - -// Type returns the type of test that can be run by this test runner. -func (r *runner) Type() testrunner.TestType { - return TestType -} - -// String returns the human-friendly name of the test runner. -func (r *runner) String() string { - return "pipeline" +type PipelineRunnerOptions struct { + Profile *profile.Profile + DeferCleanup time.Duration + API *elasticsearch.API + PackageRootPath string + TestFolder testrunner.TestFolder + GenerateTestResult bool + WithCoverage bool + CoverageType string } -// Run runs the pipeline tests defined under the given folder -func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]testrunner.TestResult, error) { - r.options = options +func NewPipelineRunner(options PipelineRunnerOptions) (*runner, error) { + r := runner{ + profile: options.Profile, + deferCleanup: options.DeferCleanup, + esAPI: options.API, + packageRootPath: options.PackageRootPath, + testFolder: options.TestFolder, + generateTestResult: options.GenerateTestResult, + withCoverage: options.WithCoverage, + coverageType: options.CoverageType, + } - stackConfig, err := stack.LoadConfig(r.options.Profile) + stackConfig, err := stack.LoadConfig(r.profile) if err != nil { return nil, err } - provider, err := stack.BuildProvider(stackConfig.Provider, r.options.Profile) + provider, err := stack.BuildProvider(stackConfig.Provider, r.profile) if err != nil { return nil, fmt.Errorf("failed to build stack provider: %w", err) } @@ -91,21 +101,44 @@ func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]tes r.runCompareResults = false } } + return &r, nil +} +type IngestPipelineReroute struct { + Description string `yaml:"description"` + Processors []map[string]ingest.RerouteProcessor `yaml:"processors"` + AdditionalFields map[string]interface{} `yaml:",inline"` +} + +// Ensures that runner implements testrunner.TestRunner interface +var _ testrunner.TestRunner = new(runner) + +// Type returns the type of test that can be run by this test runner. +func (r *runner) Type() testrunner.TestType { + return TestType +} + +// String returns the human-friendly name of the test runner. +func (r *runner) String() string { + return "pipeline" +} + +// Run runs the pipeline tests defined under the given folder +func (r *runner) Run(ctx context.Context) ([]testrunner.TestResult, error) { return r.run(ctx) } // TearDown shuts down the pipeline test runner. func (r *runner) TearDown(ctx context.Context) error { - if r.options.DeferCleanup > 0 { - logger.Debugf("Waiting for %s before cleanup...", r.options.DeferCleanup) + if r.deferCleanup > 0 { + logger.Debugf("Waiting for %s before cleanup...", r.deferCleanup) select { - case <-time.After(r.options.DeferCleanup): + case <-time.After(r.deferCleanup): case <-ctx.Done(): } } - if err := ingest.UninstallPipelines(ctx, r.options.API, r.pipelines); err != nil { + if err := ingest.UninstallPipelines(ctx, r.esAPI, r.pipelines); err != nil { return fmt.Errorf("uninstalling ingest pipelines failed: %w", err) } return nil @@ -117,11 +150,11 @@ func (r *runner) run(ctx context.Context) ([]testrunner.TestResult, error) { return nil, fmt.Errorf("listing test case definitions failed: %w", err) } - if r.options.API == nil { + if r.esAPI == nil { return nil, errors.New("missing Elasticsearch client") } - dataStreamPath, found, err := packages.FindDataStreamRootForPath(r.options.TestFolder.Path) + dataStreamPath, found, err := packages.FindDataStreamRootForPath(r.testFolder.Path) if err != nil { return nil, fmt.Errorf("locating data_stream root failed: %w", err) } @@ -131,17 +164,17 @@ func (r *runner) run(ctx context.Context) ([]testrunner.TestResult, error) { startTime := time.Now() var entryPipeline string - entryPipeline, r.pipelines, err = ingest.InstallDataStreamPipelines(r.options.API, dataStreamPath) + entryPipeline, r.pipelines, err = ingest.InstallDataStreamPipelines(r.esAPI, dataStreamPath) if err != nil { return nil, fmt.Errorf("installing ingest pipelines failed: %w", err) } - pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath) + pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.packageRootPath) if err != nil { return nil, fmt.Errorf("failed to read manifest: %w", err) } - dsManifest, err := packages.ReadDataStreamManifestFromPackageRoot(r.options.PackageRootPath, r.options.TestFolder.DataStream) + dsManifest, err := packages.ReadDataStreamManifestFromPackageRoot(r.packageRootPath, r.testFolder.DataStream) if err != nil { return nil, fmt.Errorf("failed to read data stream manifest: %w", err) } @@ -164,7 +197,7 @@ func (r *runner) run(ctx context.Context) ([]testrunner.TestResult, error) { if len(expectedDatasets) == 0 { expectedDataset := dsManifest.Dataset if expectedDataset == "" { - expectedDataset = pkgManifest.Name + "." + r.options.TestFolder.DataStream + expectedDataset = pkgManifest.Name + "." + r.testFolder.DataStream } expectedDatasets = []string{expectedDataset} } @@ -201,7 +234,7 @@ func (r *runner) checkElasticsearchLogs(ctx context.Context, startTesting time.T testingTime := startTesting.Truncate(time.Second) dumpOptions := stack.DumpOptions{ - Profile: r.options.Profile, + Profile: r.profile, Services: []string{"elasticsearch"}, Since: testingTime, } @@ -251,8 +284,8 @@ func (r *runner) checkElasticsearchLogs(ctx context.Context, startTesting time.T tr := testrunner.TestResult{ TestType: TestType, Name: "(ingest pipeline warnings)", - Package: r.options.TestFolder.Package, - DataStream: r.options.TestFolder.DataStream, + Package: r.testFolder.Package, + DataStream: r.testFolder.DataStream, TimeElapsed: time.Since(startTime), } @@ -268,12 +301,12 @@ func (r *runner) checkElasticsearchLogs(ctx context.Context, startTesting time.T func (r *runner) runTestCase(ctx context.Context, testCaseFile string, dsPath string, dsType string, pipeline string, validatorOptions []fields.ValidatorOption) (testrunner.TestResult, error) { tr := testrunner.TestResult{ TestType: TestType, - Package: r.options.TestFolder.Package, - DataStream: r.options.TestFolder.DataStream, + Package: r.testFolder.Package, + DataStream: r.testFolder.DataStream, } startTime := time.Now() - tc, err := loadTestCaseFile(r.options.TestFolder.Path, testCaseFile) + tc, err := loadTestCaseFile(r.testFolder.Path, testCaseFile) if err != nil { err := fmt.Errorf("loading test case failed: %w", err) tr.ErrorMsg = err.Error() @@ -283,15 +316,15 @@ func (r *runner) runTestCase(ctx context.Context, testCaseFile string, dsPath st if tc.config.Skip != nil { logger.Warnf("skipping %s test for %s/%s: %s (details: %s)", - TestType, r.options.TestFolder.Package, r.options.TestFolder.DataStream, + TestType, r.testFolder.Package, r.testFolder.DataStream, tc.config.Skip.Reason, tc.config.Skip.Link.String()) tr.Skipped = tc.config.Skip return tr, nil } - simulateDataStream := dsType + "-" + r.options.TestFolder.Package + "." + r.options.TestFolder.DataStream + "-default" - processedEvents, err := ingest.SimulatePipeline(ctx, r.options.API, pipeline, tc.events, simulateDataStream) + simulateDataStream := dsType + "-" + r.testFolder.Package + "." + r.testFolder.DataStream + "-default" + processedEvents, err := ingest.SimulatePipeline(ctx, r.esAPI, pipeline, tc.events, simulateDataStream) if err != nil { err := fmt.Errorf("simulating pipeline processing failed: %w", err) tr.ErrorMsg = err.Error() @@ -321,8 +354,13 @@ func (r *runner) runTestCase(ctx context.Context, testCaseFile string, dsPath st return tr, nil } - if r.options.WithCoverage { - tr.Coverage, err = GetPipelineCoverage(r.options, r.pipelines) + if r.withCoverage { + tr.Coverage, err = getPipelineCoverage(PipelineRunnerOptions{ + TestFolder: r.testFolder, + API: r.esAPI, + PackageRootPath: r.packageRootPath, + CoverageType: r.coverageType, + }, r.pipelines) if err != nil { return tr, fmt.Errorf("error calculating pipeline coverage: %w", err) } @@ -332,9 +370,9 @@ func (r *runner) runTestCase(ctx context.Context, testCaseFile string, dsPath st } func (r *runner) listTestCaseFiles() ([]string, error) { - fis, err := os.ReadDir(r.options.TestFolder.Path) + fis, err := os.ReadDir(r.testFolder.Path) if err != nil { - return nil, fmt.Errorf("reading pipeline tests failed (path: %s): %w", r.options.TestFolder.Path, err) + return nil, fmt.Errorf("reading pipeline tests failed (path: %s): %w", r.testFolder.Path, err) } var files []string @@ -393,9 +431,9 @@ func loadTestCaseFile(testFolderPath, testCaseFile string) (*testCase, error) { } func (r *runner) verifyResults(testCaseFile string, config *testConfig, result *testResult, fieldsValidator *fields.Validator) error { - testCasePath := filepath.Join(r.options.TestFolder.Path, testCaseFile) + testCasePath := filepath.Join(r.testFolder.Path, testCaseFile) - manifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath) + manifest, err := packages.ReadPackageManifestFromPackageRoot(r.packageRootPath) if err != nil { return fmt.Errorf("failed to read package manifest: %w", err) } @@ -404,7 +442,7 @@ func (r *runner) verifyResults(testCaseFile string, config *testConfig, result * return fmt.Errorf("failed to parse package format version %q: %w", manifest.SpecVersion, err) } - if r.options.GenerateTestResult { + if r.generateTestResult { err := writeTestResult(testCasePath, result, *specVersion) if err != nil { return fmt.Errorf("writing test result failed: %w", err) @@ -548,7 +586,3 @@ func checkErrorMessage(event json.RawMessage) error { return fmt.Errorf("unexpected pipeline error (unexpected error.message type %T): %[1]v", m) } } - -func init() { - testrunner.RegisterRunner(&runner{}) -} diff --git a/internal/testrunner/runners/policy/policy.go b/internal/testrunner/runners/policy/policy.go index d9b785345..0183bbff4 100644 --- a/internal/testrunner/runners/policy/policy.go +++ b/internal/testrunner/runners/policy/policy.go @@ -18,11 +18,11 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/elastic-package/internal/common" - "github.com/elastic/elastic-package/internal/testrunner" + "github.com/elastic/elastic-package/internal/kibana" ) -func dumpExpectedAgentPolicy(ctx context.Context, options testrunner.TestOptions, testPath string, policyID string, expectedRevision int) error { - policy, err := options.KibanaClient.DownloadPolicy(ctx, policyID) +func dumpExpectedAgentPolicy(ctx context.Context, kibanaClient *kibana.Client, testPath string, policyID string) error { + policy, err := kibanaClient.DownloadPolicy(ctx, policyID) if err != nil { return fmt.Errorf("failed to download policy %q: %w", policyID, err) } @@ -40,8 +40,8 @@ func dumpExpectedAgentPolicy(ctx context.Context, options testrunner.TestOptions return nil } -func assertExpectedAgentPolicy(ctx context.Context, options testrunner.TestOptions, testPath string, policyID string, expectedRevision int) error { - policy, err := options.KibanaClient.DownloadPolicy(ctx, policyID) +func assertExpectedAgentPolicy(ctx context.Context, kibanaClient *kibana.Client, testPath string, policyID string) error { + policy, err := kibanaClient.DownloadPolicy(ctx, policyID) if err != nil { return fmt.Errorf("failed to download policy %q: %w", policyID, err) } diff --git a/internal/testrunner/runners/policy/runner.go b/internal/testrunner/runners/policy/runner.go index 187cf5e1d..9958de3ed 100644 --- a/internal/testrunner/runners/policy/runner.go +++ b/internal/testrunner/runners/policy/runner.go @@ -12,6 +12,7 @@ import ( "github.com/elastic/go-resource" + "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/resources" "github.com/elastic/elastic-package/internal/testrunner" @@ -21,11 +22,28 @@ const ( TestType testrunner.TestType = "policy" ) -func init() { - testrunner.RegisterRunner(&runner{}) +type runner struct { + testFolder testrunner.TestFolder + packageRootPath string + generateTestResult bool + kibanaClient *kibana.Client } -type runner struct{} +type PolicyRunnerOptions struct { + TestFolder testrunner.TestFolder + KibanaClient *kibana.Client + PackageRootPath string + GenerateTestResult bool +} + +func NewPolicyRunner(options PolicyRunnerOptions) *runner { + return &runner{ + kibanaClient: options.KibanaClient, + testFolder: options.TestFolder, + packageRootPath: options.PackageRootPath, + generateTestResult: options.GenerateTestResult, + } +} func (r *runner) Type() testrunner.TestType { return TestType @@ -35,24 +53,24 @@ func (r *runner) String() string { return string(TestType) } -func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]testrunner.TestResult, error) { +func (r *runner) Run(ctx context.Context) ([]testrunner.TestResult, error) { manager := resources.NewManager() manager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{ - Client: options.KibanaClient, + Client: r.kibanaClient, }) - cleanup, err := r.setupSuite(ctx, manager, options) + cleanup, err := r.setupSuite(ctx, manager) if err != nil { return nil, fmt.Errorf("failed to setup test runner: %w", err) } var results []testrunner.TestResult - tests, err := filepath.Glob(filepath.Join(options.TestFolder.Path, "test-*.yml")) + tests, err := filepath.Glob(filepath.Join(r.testFolder.Path, "test-*.yml")) if err != nil { - return nil, fmt.Errorf("failed to look for test files in %s: %w", options.TestFolder.Path, err) + return nil, fmt.Errorf("failed to look for test files in %s: %w", r.testFolder.Path, err) } for _, test := range tests { - result, err := r.runTest(ctx, manager, options, test) + result, err := r.runTest(ctx, manager, test) if err != nil { logger.Error(err) } @@ -67,12 +85,12 @@ func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]tes return results, nil } -func (r *runner) runTest(ctx context.Context, manager *resources.Manager, options testrunner.TestOptions, testPath string) ([]testrunner.TestResult, error) { +func (r *runner) runTest(ctx context.Context, manager *resources.Manager, testPath string) ([]testrunner.TestResult, error) { result := testrunner.NewResultComposer(testrunner.TestResult{ TestType: TestType, Name: filepath.Base(testPath), - Package: options.TestFolder.Package, - DataStream: options.TestFolder.DataStream, + Package: r.testFolder.Package, + DataStream: r.testFolder.DataStream, }) testConfig, err := readTestConfig(testPath) @@ -86,9 +104,9 @@ func (r *runner) runTest(ctx context.Context, manager *resources.Manager, option Namespace: "ep", PackagePolicies: []resources.FleetPackagePolicy{ { - Name: testName + "-" + options.TestFolder.Package, - RootPath: options.PackageRootPath, - DataStreamName: options.TestFolder.DataStream, + Name: testName + "-" + r.testFolder.Package, + RootPath: r.packageRootPath, + DataStreamName: r.testFolder.DataStream, InputName: testConfig.Input, Vars: testConfig.Vars, DataStreamVars: testConfig.DataStream.Vars, @@ -98,14 +116,10 @@ func (r *runner) runTest(ctx context.Context, manager *resources.Manager, option resources := resource.Resources{&policy} _, testErr := manager.ApplyCtx(ctx, resources) if testErr == nil { - // Revision 1 on creation, plus one revision for each attached policy. - // In some cases the revision 2 with the agent policy disappears if there is some - // issue with the final policy. - expectedRevision := 1 + len(policy.PackagePolicies) - if options.GenerateTestResult { - testErr = dumpExpectedAgentPolicy(ctx, options, testPath, policy.ID, expectedRevision) + if r.generateTestResult { + testErr = dumpExpectedAgentPolicy(ctx, r.kibanaClient, testPath, policy.ID) } else { - testErr = assertExpectedAgentPolicy(ctx, options, testPath, policy.ID, expectedRevision) + testErr = assertExpectedAgentPolicy(ctx, r.kibanaClient, testPath, policy.ID) } } @@ -130,9 +144,9 @@ func testNameFromPath(path string) string { return strings.TrimSuffix(filepath.Base(path), ext) } -func (r *runner) setupSuite(ctx context.Context, manager *resources.Manager, options testrunner.TestOptions) (cleanup func(ctx context.Context) error, err error) { +func (r *runner) setupSuite(ctx context.Context, manager *resources.Manager) (cleanup func(ctx context.Context) error, err error) { packageResource := resources.FleetPackage{ - RootPath: options.PackageRootPath, + RootPath: r.packageRootPath, } setupResources := resources.Resources{ &packageResource, diff --git a/internal/testrunner/runners/static/runner.go b/internal/testrunner/runners/static/runner.go index 010f1256f..5e843059e 100644 --- a/internal/testrunner/runners/static/runner.go +++ b/internal/testrunner/runners/static/runner.go @@ -22,16 +22,26 @@ import ( const sampleEventJSON = "sample_event.json" type runner struct { - options testrunner.TestOptions + testFolder testrunner.TestFolder + packageRootPath string } -// Ensures that runner implements testrunner.TestRunner interface -var _ testrunner.TestRunner = new(runner) +type StaticRunnerOptions struct { + TestFolder testrunner.TestFolder + PackageRootPath string +} -func init() { - testrunner.RegisterRunner(&runner{}) +func NewStaticRunner(options StaticRunnerOptions) *runner { + runner := runner{ + testFolder: options.TestFolder, + packageRootPath: options.PackageRootPath, + } + return &runner } +// Ensures that runner implements testrunner.TestRunner interface +var _ testrunner.TestRunner = new(runner) + const ( // TestType defining asset loading tests TestType testrunner.TestType = "static" @@ -45,45 +55,44 @@ func (r runner) String() string { return "static files" } -func (r runner) Run(ctx context.Context, options testrunner.TestOptions) ([]testrunner.TestResult, error) { - r.options = options +func (r runner) Run(ctx context.Context) ([]testrunner.TestResult, error) { return r.run(ctx) } func (r runner) run(ctx context.Context) ([]testrunner.TestResult, error) { result := testrunner.NewResultComposer(testrunner.TestResult{ TestType: TestType, - Package: r.options.TestFolder.Package, - DataStream: r.options.TestFolder.DataStream, + Package: r.testFolder.Package, + DataStream: r.testFolder.DataStream, }) - testConfig, err := newConfig(r.options.TestFolder.Path) + testConfig, err := newConfig(r.testFolder.Path) if err != nil { return result.WithError(fmt.Errorf("unable to load asset loading test config file: %w", err)) } if testConfig != nil && testConfig.Skip != nil { logger.Warnf("skipping %s test for %s: %s (details: %s)", - TestType, r.options.TestFolder.Package, + TestType, r.testFolder.Package, testConfig.Skip.Reason, testConfig.Skip.Link.String()) return result.WithSkip(testConfig.Skip) } - pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath) + pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.packageRootPath) if err != nil { return result.WithError(fmt.Errorf("failed to read manifest: %w", err)) } // join together results from verifyStreamConfig and verifySampleEvent - return append(r.verifyStreamConfig(ctx, r.options.PackageRootPath), r.verifySampleEvent(pkgManifest)...), nil + return append(r.verifyStreamConfig(ctx, r.packageRootPath), r.verifySampleEvent(pkgManifest)...), nil } func (r runner) verifyStreamConfig(ctx context.Context, packageRootPath string) []testrunner.TestResult { resultComposer := testrunner.NewResultComposer(testrunner.TestResult{ Name: "Verify benchmark config", TestType: TestType, - Package: r.options.TestFolder.Package, - DataStream: r.options.TestFolder.DataStream, + Package: r.testFolder.Package, + DataStream: r.testFolder.DataStream, }) withOpts := []stream.OptionFunc{ @@ -93,7 +102,7 @@ func (r runner) verifyStreamConfig(ctx context.Context, packageRootPath string) ctx, stop := signal.Enable(ctx, logger.Info) defer stop() - hasBenchmark, err := stream.StaticValidation(ctx, stream.NewOptions(withOpts...), r.options.TestFolder.DataStream) + hasBenchmark, err := stream.StaticValidation(ctx, stream.NewOptions(withOpts...), r.testFolder.DataStream) if err != nil { results, _ := resultComposer.WithError(err) return results @@ -111,8 +120,8 @@ func (r runner) verifySampleEvent(pkgManifest *packages.PackageManifest) []testr resultComposer := testrunner.NewResultComposer(testrunner.TestResult{ Name: "Verify " + sampleEventJSON, TestType: TestType, - Package: r.options.TestFolder.Package, - DataStream: r.options.TestFolder.DataStream, + Package: r.testFolder.Package, + DataStream: r.testFolder.DataStream, }) sampleEventPath, found, err := r.getSampleEventPath() @@ -162,14 +171,14 @@ func (r runner) verifySampleEvent(pkgManifest *packages.PackageManifest) []testr func (r runner) getSampleEventPath() (string, bool, error) { var sampleEventPath string - if r.options.TestFolder.DataStream != "" { + if r.testFolder.DataStream != "" { sampleEventPath = filepath.Join( - r.options.PackageRootPath, + r.packageRootPath, "data_stream", - r.options.TestFolder.DataStream, + r.testFolder.DataStream, sampleEventJSON) } else { - sampleEventPath = filepath.Join(r.options.PackageRootPath, sampleEventJSON) + sampleEventPath = filepath.Join(r.packageRootPath, sampleEventJSON) } _, err := os.Stat(sampleEventPath) if errors.Is(err, os.ErrNotExist) { @@ -182,7 +191,7 @@ func (r runner) getSampleEventPath() (string, bool, error) { } func (r runner) getExpectedDatasets(pkgManifest *packages.PackageManifest) ([]string, error) { - dsName := r.options.TestFolder.DataStream + dsName := r.testFolder.DataStream if dsName == "" { // TODO: This should return the package name plus the policy name, but we don't know // what policy created this event, so we cannot reliably know it here. Skip the check @@ -190,7 +199,7 @@ func (r runner) getExpectedDatasets(pkgManifest *packages.PackageManifest) ([]st return nil, nil } - dataStreamManifest, err := packages.ReadDataStreamManifestFromPackageRoot(r.options.PackageRootPath, dsName) + dataStreamManifest, err := packages.ReadDataStreamManifestFromPackageRoot(r.packageRootPath, dsName) if err != nil { return nil, fmt.Errorf("failed to read data stream manifest: %w", err) } diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index caaecd4ff..33bee6ffe 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -32,6 +32,7 @@ import ( "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/multierror" "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/resources" "github.com/elastic/elastic-package/internal/servicedeployer" "github.com/elastic/elastic-package/internal/stack" @@ -76,10 +77,6 @@ const ( DevDeployDir = "_dev/deploy" ) -func init() { - testrunner.RegisterRunner(&runner{}) -} - const ( // TestType defining system tests TestType testrunner.TestType = "system" @@ -131,7 +128,25 @@ var ( ) type runner struct { - options testrunner.TestOptions + profile *profile.Profile + testFolder testrunner.TestFolder + packageRootPath string + generateTestResult bool + esAPI *elasticsearch.API + kibanaClient *kibana.Client + + runIndependentElasticAgent bool + + deferCleanup time.Duration + serviceVariant string + withCoverage bool + coverageType string + + configFilePath string + runSetup bool + runTearDown bool + runTestsOnly bool + pipelines []ingest.Pipeline dataStreamPath string @@ -153,6 +168,49 @@ type runner struct { wipeDataStreamHandler func(context.Context) error } +type SystemRunnerOptions struct { + Profile *profile.Profile + TestFolder testrunner.TestFolder + PackageRootPath string + GenerateTestResult bool + API *elasticsearch.API + KibanaClient *kibana.Client + + RunIndependentElasticAgent bool + + DeferCleanup time.Duration + ServiceVariant string + WithCoverage bool + CoverageType string + + ConfigFilePath string + RunSetup bool + RunTearDown bool + RunTestsOnly bool +} + +func NewSystemRunner(options SystemRunnerOptions) *runner { + r := runner{ + profile: options.Profile, + testFolder: options.TestFolder, + packageRootPath: options.PackageRootPath, + generateTestResult: options.GenerateTestResult, + esAPI: options.API, + kibanaClient: options.KibanaClient, + runIndependentElasticAgent: options.RunIndependentElasticAgent, + deferCleanup: options.DeferCleanup, + serviceVariant: options.ServiceVariant, + withCoverage: options.WithCoverage, + coverageType: options.CoverageType, + configFilePath: options.ConfigFilePath, + runSetup: options.RunSetup, + runTestsOnly: options.RunTestsOnly, + runTearDown: options.RunTearDown, + } + // TODO: check if logic in initRun could be moved to this constructor + return &r +} + // Ensures that runner implements testrunner.TestRunner interface var _ testrunner.TestRunner = new(runner) @@ -167,9 +225,8 @@ func (r *runner) String() string { } // Run runs the system tests defined under the given folder -func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]testrunner.TestResult, error) { - r.options = options - if !r.options.RunSetup && !r.options.RunTearDown && !r.options.RunTestsOnly { +func (r *runner) Run(ctx context.Context) ([]testrunner.TestResult, error) { + if !r.runSetup && !r.runTearDown && !r.runTestsOnly { return r.run(ctx) } @@ -178,7 +235,7 @@ func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]tes return result.WithError(err) } - if r.options.RunSetup { + if r.runSetup { // variant information in runTestOnly or runTearDown modes is retrieved from serviceOptions (file in setup dir) if len(r.variants) > 1 { return result.WithError(fmt.Errorf("a variant must be selected or trigger the test in no-variant mode (available variants: %s)", strings.Join(r.variants, ", "))) @@ -190,27 +247,27 @@ func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]tes _, err := os.Stat(r.serviceStateFilePath) logger.Debugf("Service state data exists in %s: %v", r.serviceStateFilePath, !os.IsNotExist(err)) - if r.options.RunSetup && !os.IsNotExist(err) { + if r.runSetup && !os.IsNotExist(err) { return result.WithError(fmt.Errorf("failed to run --setup, required to tear down previous setup")) } - if r.options.RunTestsOnly && os.IsNotExist(err) { + if r.runTestsOnly && os.IsNotExist(err) { return result.WithError(fmt.Errorf("failed to run tests with --no-provision, setup first with --setup")) } - if r.options.RunTearDown && os.IsNotExist(err) { + if r.runTearDown && os.IsNotExist(err) { return result.WithError(fmt.Errorf("failed to run --tear-down, setup not found")) } var serviceStateData ServiceState - if !r.options.RunSetup { + if !r.runSetup { serviceStateData, err = r.readServiceStateData() if err != nil { return result.WithError(fmt.Errorf("failed to read service state: %w", err)) } } - configFile := r.options.ConfigFilePath + configFile := r.configFilePath variant := r.variants[0] - if r.options.RunTestsOnly || r.options.RunTearDown { + if r.runTestsOnly || r.runTearDown { configFile = serviceStateData.ConfigFilePath variant = serviceStateData.VariantName @@ -231,17 +288,17 @@ func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]tes resultName := "" switch { - case r.options.RunSetup: + case r.runSetup: resultName = "setup" - case r.options.RunTearDown: + case r.runTearDown: resultName = "teardown" - case r.options.RunTestsOnly: + case r.runTestsOnly: resultName = "tests" } result = r.newResult(fmt.Sprintf("%s - %s", resultName, testConfig.Name())) scenario, err := r.prepareScenario(ctx, testConfig, svcInfo) - if r.options.RunSetup && err != nil { + if r.runSetup && err != nil { tdErr := r.tearDownTest(ctx) if tdErr != nil { logger.Errorf("failed to tear down runner: %s", tdErr.Error()) @@ -254,14 +311,14 @@ func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]tes return result.WithError(err) } - if r.options.RunTestsOnly { + if r.runTestsOnly { if err != nil { return result.WithError(fmt.Errorf("failed to prepare scenario: %w", err)) } return r.validateTestScenario(ctx, result, scenario, testConfig) } - if r.options.RunTearDown { + if r.runTearDown { if err != nil { logger.Errorf("failed to prepare scenario: %s", err.Error()) logger.Errorf("continue with the tear down process") @@ -286,7 +343,7 @@ type resourcesOptions struct { func (r *runner) resources(opts resourcesOptions) resources.Resources { return resources.Resources{ &resources.FleetPackage{ - RootPath: r.options.PackageRootPath, + RootPath: r.packageRootPath, Absent: !opts.installedPackage, Force: opts.installedPackage, // Force re-installation, in case there are code changes in the same package version. }, @@ -295,51 +352,51 @@ func (r *runner) resources(opts resourcesOptions) resources.Resources { func (r *runner) createAgentOptions(policyName string) agentdeployer.FactoryOptions { return agentdeployer.FactoryOptions{ - Profile: r.options.Profile, - PackageRootPath: r.options.PackageRootPath, + Profile: r.profile, + PackageRootPath: r.packageRootPath, DataStreamRootPath: r.dataStreamPath, DevDeployDir: DevDeployDir, Type: agentdeployer.TypeTest, StackVersion: r.stackVersion.Version(), - PackageName: r.options.TestFolder.Package, - DataStream: r.options.TestFolder.DataStream, + PackageName: r.testFolder.Package, + DataStream: r.testFolder.DataStream, PolicyName: policyName, - RunTearDown: r.options.RunTearDown, - RunTestsOnly: r.options.RunTestsOnly, - RunSetup: r.options.RunSetup, + RunTearDown: r.runTearDown, + RunTestsOnly: r.runTestsOnly, + RunSetup: r.runSetup, } } func (r *runner) createServiceOptions(variantName string) servicedeployer.FactoryOptions { return servicedeployer.FactoryOptions{ - Profile: r.options.Profile, - PackageRootPath: r.options.PackageRootPath, + Profile: r.profile, + PackageRootPath: r.packageRootPath, DataStreamRootPath: r.dataStreamPath, DevDeployDir: DevDeployDir, Variant: variantName, Type: servicedeployer.TypeTest, StackVersion: r.stackVersion.Version(), - RunTearDown: r.options.RunTearDown, - RunTestsOnly: r.options.RunTestsOnly, - RunSetup: r.options.RunSetup, - DeployIndependentAgent: r.options.RunIndependentElasticAgent, + RunTearDown: r.runTearDown, + RunTestsOnly: r.runTestsOnly, + RunSetup: r.runSetup, + DeployIndependentAgent: r.runIndependentElasticAgent, } } func (r *runner) createAgentInfo(policy *kibana.Policy, config *testConfig, runID string, agentManifest packages.Agent) (agentdeployer.AgentInfo, error) { var info agentdeployer.AgentInfo - info.Name = r.options.TestFolder.Package + info.Name = r.testFolder.Package info.Logs.Folder.Agent = ServiceLogsAgentDir info.Test.RunID = runID - folderName := fmt.Sprintf("agent-%s", r.options.TestFolder.Package) - if r.options.TestFolder.DataStream != "" { - folderName = fmt.Sprintf("%s-%s", folderName, r.options.TestFolder.DataStream) + folderName := fmt.Sprintf("agent-%s", r.testFolder.Package) + if r.testFolder.DataStream != "" { + folderName = fmt.Sprintf("%s-%s", folderName, r.testFolder.DataStream) } folderName = fmt.Sprintf("%s-%s", folderName, runID) - dirPath, err := agentdeployer.CreateServiceLogsDir(r.options.Profile, folderName) + dirPath, err := agentdeployer.CreateServiceLogsDir(r.profile, folderName) if err != nil { return agentdeployer.AgentInfo{}, fmt.Errorf("failed to create service logs dir: %w", err) } @@ -362,12 +419,12 @@ func (r *runner) createAgentInfo(policy *kibana.Policy, config *testConfig, runI func (r *runner) createServiceInfo() (servicedeployer.ServiceInfo, error) { var svcInfo servicedeployer.ServiceInfo - svcInfo.Name = r.options.TestFolder.Package + svcInfo.Name = r.testFolder.Package svcInfo.Logs.Folder.Local = r.locationManager.ServiceLogDir() svcInfo.Logs.Folder.Agent = ServiceLogsAgentDir svcInfo.Test.RunID = common.CreateTestRunID() - if r.options.RunTearDown || r.options.RunTestsOnly { + if r.runTearDown || r.runTestsOnly { logger.Debug("Skip creating output directory") } else { outputDir, err := servicedeployer.CreateOutputDir(r.locationManager, svcInfo.Test.RunID) @@ -388,10 +445,10 @@ func (r *runner) TearDown(ctx context.Context) error { } func (r *runner) tearDownTest(ctx context.Context) error { - if r.options.DeferCleanup > 0 { - logger.Debugf("waiting for %s before tearing down...", r.options.DeferCleanup) + if r.deferCleanup > 0 { + logger.Debugf("waiting for %s before tearing down...", r.deferCleanup) select { - case <-time.After(r.options.DeferCleanup): + case <-time.After(r.deferCleanup): case <-ctx.Done(): } } @@ -429,7 +486,7 @@ func (r *runner) tearDownTest(ctx context.Context) error { resourcesOptions := resourcesOptions{ // Keep it installed only if we were running setup, or tests only. - installedPackage: r.options.RunSetup || r.options.RunTestsOnly, + installedPackage: r.runSetup || r.runTestsOnly, } _, err := r.resourcesManager.ApplyCtx(cleanupCtx, r.resources(resourcesOptions)) if err != nil { @@ -464,8 +521,8 @@ func (r *runner) newResult(name string) *testrunner.ResultComposer { return testrunner.NewResultComposer(testrunner.TestResult{ TestType: TestType, Name: name, - Package: r.options.TestFolder.Package, - DataStream: r.options.TestFolder.DataStream, + Package: r.testFolder.Package, + DataStream: r.testFolder.DataStream, }) } @@ -478,34 +535,34 @@ func (r *runner) initRun() error { } r.resourcesManager = resources.NewManager() - r.resourcesManager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: r.options.KibanaClient}) + r.resourcesManager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: r.kibanaClient}) - r.serviceStateFilePath = filepath.Join(testrunner.StateFolderPath(r.options.Profile.ProfilePath), testrunner.ServiceStateFileName) + r.serviceStateFilePath = filepath.Join(testrunner.StateFolderPath(r.profile.ProfilePath), testrunner.ServiceStateFileName) - r.dataStreamPath, found, err = packages.FindDataStreamRootForPath(r.options.TestFolder.Path) + r.dataStreamPath, found, err = packages.FindDataStreamRootForPath(r.testFolder.Path) if err != nil { return fmt.Errorf("locating data stream root failed: %w", err) } if found { - logger.Debugf("Running system tests for data stream %q", r.options.TestFolder.DataStream) + logger.Debugf("Running system tests for data stream %q", r.testFolder.DataStream) } else { logger.Debug("Running system tests for package") } - if r.options.API == nil { + if r.esAPI == nil { return errors.New("missing Elasticsearch client") } - if r.options.KibanaClient == nil { + if r.kibanaClient == nil { return errors.New("missing Kibana client") } - r.stackVersion, err = r.options.KibanaClient.Version() + r.stackVersion, err = r.kibanaClient.Version() if err != nil { return fmt.Errorf("cannot request Kibana version: %w", err) } devDeployPath, err := servicedeployer.FindDevDeployPath(servicedeployer.FactoryOptions{ - PackageRootPath: r.options.PackageRootPath, + PackageRootPath: r.packageRootPath, DataStreamRootPath: r.dataStreamPath, DevDeployDir: DevDeployDir, }) @@ -521,23 +578,23 @@ func (r *runner) initRun() error { } r.variants = r.selectVariants(variantsFile) } - if r.options.ServiceVariant != "" && len(r.variants) == 0 { - return fmt.Errorf("not found variant definition %q", r.options.ServiceVariant) + if r.serviceVariant != "" && len(r.variants) == 0 { + return fmt.Errorf("not found variant definition %q", r.serviceVariant) } - if r.options.ConfigFilePath != "" { - allCfgFiles, err := listConfigFiles(filepath.Dir(r.options.ConfigFilePath)) + if r.configFilePath != "" { + allCfgFiles, err := listConfigFiles(filepath.Dir(r.configFilePath)) if err != nil { return fmt.Errorf("failed listing test case config cfgFiles: %w", err) } - baseFile := filepath.Base(r.options.ConfigFilePath) + baseFile := filepath.Base(r.configFilePath) for _, cfg := range allCfgFiles { if cfg == baseFile { r.cfgFiles = append(r.cfgFiles, baseFile) } } } else { - r.cfgFiles, err = listConfigFiles(r.options.TestFolder.Path) + r.cfgFiles, err = listConfigFiles(r.testFolder.Path) if err != nil { return fmt.Errorf("failed listing test case config cfgFiles: %w", err) } @@ -573,19 +630,19 @@ func (r *runner) run(ctx context.Context) (results []testrunner.TestResult, err } defer os.RemoveAll(tempDir) - stackConfig, err := stack.LoadConfig(r.options.Profile) + stackConfig, err := stack.LoadConfig(r.profile) if err != nil { return nil, err } - provider, err := stack.BuildProvider(stackConfig.Provider, r.options.Profile) + provider, err := stack.BuildProvider(stackConfig.Provider, r.profile) if err != nil { return nil, fmt.Errorf("failed to build stack provider: %w", err) } dumpOptions := stack.DumpOptions{ Output: tempDir, - Profile: r.options.Profile, + Profile: r.profile, } dump, err := provider.Dump(context.WithoutCancel(ctx), dumpOptions) if err != nil { @@ -607,7 +664,7 @@ func (r *runner) runTestPerVariant(ctx context.Context, result *testrunner.Resul return result.WithError(err) } - configFile := filepath.Join(r.options.TestFolder.Path, cfgFile) + configFile := filepath.Join(r.testFolder.Path, cfgFile) testConfig, err := newConfig(configFile, svcInfo, variantName) if err != nil { return nil, fmt.Errorf("unable to load system test case file '%s': %w", configFile, err) @@ -627,9 +684,9 @@ func (r *runner) runTestPerVariant(ctx context.Context, result *testrunner.Resul } func (r *runner) isSyntheticsEnabled(ctx context.Context, dataStream, componentTemplatePackage string) (bool, error) { - resp, err := r.options.API.Cluster.GetComponentTemplate( - r.options.API.Cluster.GetComponentTemplate.WithContext(ctx), - r.options.API.Cluster.GetComponentTemplate.WithName(componentTemplatePackage), + resp, err := r.esAPI.Cluster.GetComponentTemplate( + r.esAPI.Cluster.GetComponentTemplate.WithContext(ctx), + r.esAPI.Cluster.GetComponentTemplate.WithName(componentTemplatePackage), ) if err != nil { return false, fmt.Errorf("could not get component template %s from data stream %s: %w", componentTemplatePackage, dataStream, err) @@ -701,14 +758,14 @@ func (h hits) size() int { } func (r *runner) getDocs(ctx context.Context, dataStream string) (*hits, error) { - resp, err := r.options.API.Search( - r.options.API.Search.WithContext(ctx), - r.options.API.Search.WithIndex(dataStream), - r.options.API.Search.WithSort("@timestamp:asc"), - r.options.API.Search.WithSize(elasticsearchQuerySize), - r.options.API.Search.WithSource("true"), - r.options.API.Search.WithBody(strings.NewReader(checkFieldsBody)), - r.options.API.Search.WithIgnoreUnavailable(true), + resp, err := r.esAPI.Search( + r.esAPI.Search.WithContext(ctx), + r.esAPI.Search.WithIndex(dataStream), + r.esAPI.Search.WithSort("@timestamp:asc"), + r.esAPI.Search.WithSize(elasticsearchQuerySize), + r.esAPI.Search.WithSource("true"), + r.esAPI.Search.WithBody(strings.NewReader(checkFieldsBody)), + r.esAPI.Search.WithIgnoreUnavailable(true), ) if err != nil { return nil, fmt.Errorf("could not search data stream: %w", err) @@ -800,7 +857,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf var err error var serviceStateData ServiceState - if r.options.RunSetup { + if r.runSetup { err = r.createServiceStateDir() if err != nil { return nil, fmt.Errorf("failed to create setup services dir: %w", err) @@ -808,14 +865,14 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf } scenario := scenarioTest{} - if r.options.RunTearDown || r.options.RunTestsOnly { + if r.runTearDown || r.runTestsOnly { serviceStateData, err = r.readServiceStateData() if err != nil { return nil, fmt.Errorf("failed to read service setup data: %w", err) } } - scenario.pkgManifest, err = packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath) + scenario.pkgManifest, err = packages.ReadPackageManifestFromPackageRoot(r.packageRootPath) if err != nil { return nil, fmt.Errorf("reading package manifest failed: %w", err) } @@ -828,16 +885,16 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf // Temporarily until independent Elastic Agents are enabled by default, // enable independent Elastic Agents if package defines that requires root privileges if pkg, ds := scenario.pkgManifest, scenario.dataStreamManifest; pkg.Agent.Privileges.Root || (ds != nil && ds.Agent.Privileges.Root) { - r.options.RunIndependentElasticAgent = true + r.runIndependentElasticAgent = true } // If the environment variable is present, it always has preference over the root // privileges value (if any) defined in the manifest file v, ok := os.LookupEnv(enableIndependentAgents) if ok { - r.options.RunIndependentElasticAgent = strings.ToLower(v) == "true" + r.runIndependentElasticAgent = strings.ToLower(v) == "true" } - serviceOptions.DeployIndependentAgent = r.options.RunIndependentElasticAgent + serviceOptions.DeployIndependentAgent = r.runIndependentElasticAgent policyTemplateName := config.PolicyTemplate if policyTemplateName == "" { @@ -855,7 +912,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf // Configure package (single data stream) via Fleet APIs. var policyToTest, policyToEnroll *kibana.Policy - if r.options.RunTearDown || r.options.RunTestsOnly { + if r.runTearDown || r.runTestsOnly { policyToTest = &serviceStateData.CurrentPolicy policyToEnroll = &serviceStateData.EnrollPolicy logger.Debugf("Got policy from file: %q - %q", policyToTest.Name, policyToTest.ID) @@ -867,35 +924,35 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf testTime := time.Now().Format("20060102T15:04:05Z") policyEnroll := kibana.Policy{ - Name: fmt.Sprintf("ep-test-system-enroll-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, testTime), - Description: fmt.Sprintf("test policy created by elastic-package to enroll agent for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream), + Name: fmt.Sprintf("ep-test-system-enroll-%s-%s-%s", r.testFolder.Package, r.testFolder.DataStream, testTime), + Description: fmt.Sprintf("test policy created by elastic-package to enroll agent for data stream %s/%s", r.testFolder.Package, r.testFolder.DataStream), Namespace: "ep", } policyTest := kibana.Policy{ - Name: fmt.Sprintf("ep-test-system-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, testTime), - Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream), + Name: fmt.Sprintf("ep-test-system-%s-%s-%s", r.testFolder.Package, r.testFolder.DataStream, testTime), + Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.testFolder.Package, r.testFolder.DataStream), Namespace: "ep", } // Assign the data_output_id to the agent policy to configure the output to logstash. The value is inferred from stack/_static/kibana.yml.tmpl - if r.options.Profile.Config("stack.logstash_enabled", "false") == "true" { + if r.profile.Config("stack.logstash_enabled", "false") == "true" { policyTest.DataOutputID = "fleet-logstash-output" } - policyToTest, err = r.options.KibanaClient.CreatePolicy(ctx, policyTest) + policyToTest, err = r.kibanaClient.CreatePolicy(ctx, policyTest) if err != nil { return nil, fmt.Errorf("could not create test policy: %w", err) } - policyToEnroll, err = r.options.KibanaClient.CreatePolicy(ctx, policyEnroll) + policyToEnroll, err = r.kibanaClient.CreatePolicy(ctx, policyEnroll) if err != nil { return nil, fmt.Errorf("could not create test policy: %w", err) } } r.deleteTestPolicyHandler = func(ctx context.Context) error { logger.Debug("deleting test policies...") - if err := r.options.KibanaClient.DeletePolicy(ctx, policyToTest.ID); err != nil { + if err := r.kibanaClient.DeletePolicy(ctx, policyToTest.ID); err != nil { return fmt.Errorf("error cleaning up test policy: %w", err) } - if err := r.options.KibanaClient.DeletePolicy(ctx, policyToEnroll.ID); err != nil { + if err := r.kibanaClient.DeletePolicy(ctx, policyToEnroll.ID); err != nil { return fmt.Errorf("error cleaning up test policy: %w", err) } return nil @@ -903,7 +960,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf // policyToEnroll is used in both independent agents and agents created by servicedeployer (custom or kubernetes agents) policy := policyToEnroll - if r.options.RunTearDown || r.options.RunTestsOnly { + if r.runTearDown || r.runTestsOnly { // required in order to be able select the right agent in `checkEnrolledAgents` when // using independent agents or custom/kubernetes agents since policy data is set into `agentInfo` variable` policy = policyToTest @@ -929,7 +986,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf return nil, fmt.Errorf("unable to reload system test case configuration: %w", err) } - if r.options.RunTearDown { + if r.runTearDown { logger.Debug("Skip installing package") } else { // Install the package before creating the policy, so we control exactly what is being @@ -937,7 +994,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf logger.Debug("Installing package...") resourcesOptions := resourcesOptions{ // Install it unless we are running the tear down only. - installedPackage: !r.options.RunTearDown, + installedPackage: !r.runTearDown, } _, err = r.resourcesManager.ApplyCtx(ctx, r.resources(resourcesOptions)) if err != nil { @@ -951,10 +1008,10 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf logger.Debug("adding package data stream to test policy...") ds := createPackageDatastream(*policyToTest, *scenario.pkgManifest, policyTemplate, *scenario.dataStreamManifest, *config, svcInfo.Test.RunID) - if r.options.RunTearDown || r.options.RunTestsOnly { + if r.runTearDown || r.runTestsOnly { logger.Debug("Skip adding data stream config to policy") } else { - if err := r.options.KibanaClient.AddPackageDataStreamToPolicy(ctx, ds); err != nil { + if err := r.kibanaClient.AddPackageDataStreamToPolicy(ctx, ds); err != nil { return nil, fmt.Errorf("could not add data stream config to policy: %w", err) } } @@ -985,16 +1042,16 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf r.wipeDataStreamHandler = func(ctx context.Context) error { logger.Debugf("deleting data in data stream...") - if err := deleteDataStreamDocs(ctx, r.options.API, scenario.dataStream); err != nil { + if err := deleteDataStreamDocs(ctx, r.esAPI, scenario.dataStream); err != nil { return fmt.Errorf("error deleting data in data stream: %w", err) } return nil } switch { - case r.options.RunTearDown: + case r.runTearDown: logger.Debugf("Skipped deleting old data in data stream %q", scenario.dataStream) - case r.options.RunTestsOnly: + case r.runTestsOnly: // In this mode, service is still running and the agent is sending documents, so sometimes // cannot be guaranteed to be zero documents err := r.deleteOldDocumentsDataStreamAndWait(ctx, scenario.dataStream, false) @@ -1010,7 +1067,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf // FIXME: running per stages does not work when multiple agents are created var origPolicy kibana.Policy - agents, err := checkEnrolledAgents(ctx, r.options.KibanaClient, agentInfo, svcInfo, r.options.RunIndependentElasticAgent) + agents, err := checkEnrolledAgents(ctx, r.kibanaClient, agentInfo, svcInfo, r.runIndependentElasticAgent) if err != nil { return nil, fmt.Errorf("can't check enrolled agents: %w", err) } @@ -1019,18 +1076,18 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf r.removeAgentHandler = func(ctx context.Context) error { // When not using independent agents, service deployers like kubernetes or custom agents create new Elastic Agent - if !r.options.RunIndependentElasticAgent && !svcInfo.Agent.Independent { + if !r.runIndependentElasticAgent && !svcInfo.Agent.Independent { return nil } logger.Debug("removing agent...") - err := r.options.KibanaClient.RemoveAgent(ctx, agent) + err := r.kibanaClient.RemoveAgent(ctx, agent) if err != nil { return fmt.Errorf("failed to remove agent %q: %w", agent.ID, err) } return nil } - if r.options.RunTearDown || r.options.RunTestsOnly { + if r.runTearDown || r.runTestsOnly { origPolicy = serviceStateData.OrigPolicy logger.Debugf("Got orig policy from file: %q - %q", origPolicy.Name, origPolicy.ID) } else { @@ -1042,11 +1099,11 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf } // Assign policy to agent r.resetAgentPolicyHandler = func(ctx context.Context) error { - if r.options.RunIndependentElasticAgent { + if r.runIndependentElasticAgent { return nil } logger.Debug("reassigning original policy back to agent...") - if err := r.options.KibanaClient.AssignPolicyToAgent(ctx, agent, origPolicy); err != nil { + if err := r.kibanaClient.AssignPolicyToAgent(ctx, agent, origPolicy); err != nil { return fmt.Errorf("error reassigning original policy to agent: %w", err) } return nil @@ -1054,13 +1111,13 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf origAgent := agent origLogLevel := "" - if r.options.RunTearDown { + if r.runTearDown { logger.Debug("Skip assiging log level debug to agent") origLogLevel = serviceStateData.Agent.LocalMetadata.Elastic.Agent.LogLevel } else { logger.Debug("Set Debug log level to agent") origLogLevel = agent.LocalMetadata.Elastic.Agent.LogLevel - err = r.options.KibanaClient.SetAgentLogLevel(ctx, agent.ID, "debug") + err = r.kibanaClient.SetAgentLogLevel(ctx, agent.ID, "debug") if err != nil { return nil, fmt.Errorf("error setting log level debug for agent %s: %w", agent.ID, err) } @@ -1068,22 +1125,22 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf r.resetAgentLogLevelHandler = func(ctx context.Context) error { logger.Debugf("reassigning original log level %q back to agent...", origLogLevel) - if err := r.options.KibanaClient.SetAgentLogLevel(ctx, agent.ID, origLogLevel); err != nil { + if err := r.kibanaClient.SetAgentLogLevel(ctx, agent.ID, origLogLevel); err != nil { return fmt.Errorf("error reassigning original log level to agent: %w", err) } return nil } - if r.options.RunTearDown || r.options.RunTestsOnly { + if r.runTearDown || r.runTestsOnly { logger.Debug("Skip assiging package data stream to agent") } else { - policyWithDataStream, err := r.options.KibanaClient.GetPolicy(ctx, policyToTest.ID) + policyWithDataStream, err := r.kibanaClient.GetPolicy(ctx, policyToTest.ID) if err != nil { return nil, fmt.Errorf("could not read the policy with data stream: %w", err) } logger.Debug("assigning package data stream to agent...") - if err := r.options.KibanaClient.AssignPolicyToAgent(ctx, agent, *policyWithDataStream); err != nil { + if err := r.kibanaClient.AssignPolicyToAgent(ctx, agent, *policyWithDataStream); err != nil { return nil, fmt.Errorf("could not assign policy to agent: %w", err) } } @@ -1095,7 +1152,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf } } - if r.options.RunTearDown { + if r.runTearDown { return &scenario, nil } @@ -1162,7 +1219,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf scenario.ignoredFields = hits.IgnoredFields scenario.degradedDocs = hits.DegradedDocs - if r.options.RunSetup { + if r.runSetup { opts := scenarioStateOpts{ origPolicy: &origPolicy, enrollPolicy: policyToEnroll, @@ -1183,19 +1240,19 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOptions servicedeployer.FactoryOptions, svcInfo servicedeployer.ServiceInfo, agentInfo agentdeployer.AgentInfo, agentDeployed agentdeployer.DeployedAgent, policy *kibana.Policy, state ServiceState) (servicedeployer.DeployedService, servicedeployer.ServiceInfo, error) { logger.Debug("setting up service...") - if r.options.RunTearDown || r.options.RunTestsOnly { + if r.runTearDown || r.runTestsOnly { svcInfo.Test.RunID = state.ServiceRunID svcInfo.OutputDir = state.ServiceOutputDir } // By default using agent running in the Elastic stack - svcInfo.AgentNetworkName = stack.Network(r.options.Profile) + svcInfo.AgentNetworkName = stack.Network(r.profile) if agentDeployed != nil { svcInfo.AgentNetworkName = agentInfo.NetworkName } // Set the right folder for logs execpt for custom agents that are still deployed using "servicedeployer" - if r.options.RunIndependentElasticAgent && agentDeployed != nil { + if r.runIndependentElasticAgent && agentDeployed != nil { svcInfo.Logs.Folder.Local = agentInfo.Logs.Folder.Local } @@ -1229,11 +1286,11 @@ func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOp } func (r *runner) setupAgent(ctx context.Context, config *testConfig, state ServiceState, policy *kibana.Policy, agentManifest packages.Agent) (agentdeployer.DeployedAgent, agentdeployer.AgentInfo, error) { - if !r.options.RunIndependentElasticAgent { + if !r.runIndependentElasticAgent { return nil, agentdeployer.AgentInfo{}, nil } agentRunID := common.CreateTestRunID() - if r.options.RunTearDown || r.options.RunTestsOnly { + if r.runTearDown || r.runTestsOnly { agentRunID = state.AgentRunID } logger.Warn("setting up agent (technical preview)...") @@ -1349,7 +1406,7 @@ func (r *runner) writeScenarioState(opts scenarioStateOpts) error { func (r *runner) deleteOldDocumentsDataStreamAndWait(ctx context.Context, dataStream string, mustBeZero bool) error { logger.Debugf("Delete previous documents in data stream %q", dataStream) - if err := deleteDataStreamDocs(ctx, r.options.API, dataStream); err != nil { + if err := deleteDataStreamDocs(ctx, r.esAPI, dataStream); err != nil { return fmt.Errorf("error deleting old data in data stream: %s: %w", dataStream, err) } startHits, err := r.getDocs(ctx, dataStream) @@ -1406,7 +1463,7 @@ func (r *runner) validateTestScenario(ctx context.Context, result *testrunner.Re if expectedDatasets == nil { var expectedDataset string - if ds := r.options.TestFolder.DataStream; ds != "" { + if ds := r.testFolder.DataStream; ds != "" { expectedDataset = getDataStreamDataset(*scenario.pkgManifest, *scenario.dataStreamManifest) } else { expectedDataset = scenario.pkgManifest.Name + "." + scenario.policyTemplateName @@ -1453,7 +1510,7 @@ func (r *runner) validateTestScenario(ctx context.Context, result *testrunner.Re } // Write sample events file from first doc, if requested - if err := r.generateTestResult(docs, *specVersion); err != nil { + if err := r.generateTestResultFile(docs, *specVersion); err != nil { return result.WithError(err) } @@ -1485,7 +1542,7 @@ func (r *runner) runTest(ctx context.Context, config *testConfig, svcInfo servic if config.Skip != nil { logger.Warnf("skipping %s test for %s/%s: %s (details: %s)", - TestType, r.options.TestFolder.Package, r.options.TestFolder.DataStream, + TestType, r.testFolder.Package, r.testFolder.DataStream, config.Skip.Reason, config.Skip.Link.String()) return result.WithSkip(config.Skip) } @@ -1786,9 +1843,9 @@ func selectPolicyTemplateByName(policies []packages.PolicyTemplate, name string) } func (r *runner) checkTransforms(ctx context.Context, config *testConfig, pkgManifest *packages.PackageManifest, ds kibana.PackageDataStream, dataStream string) error { - transforms, err := packages.ReadTransformsFromPackageRoot(r.options.PackageRootPath) + transforms, err := packages.ReadTransformsFromPackageRoot(r.packageRootPath) if err != nil { - return fmt.Errorf("loading transforms for package failed (root: %s): %w", r.options.PackageRootPath, err) + return fmt.Errorf("loading transforms for package failed (root: %s): %w", r.packageRootPath, err) } for _, transform := range transforms { hasSource, err := transform.HasSource(dataStream) @@ -1844,9 +1901,9 @@ func (r *runner) checkTransforms(ctx context.Context, config *testConfig, pkgMan } func (r *runner) getTransformId(ctx context.Context, transformPattern string) (string, error) { - resp, err := r.options.API.TransformGetTransform( - r.options.API.TransformGetTransform.WithContext(ctx), - r.options.API.TransformGetTransform.WithTransformID(transformPattern), + resp, err := r.esAPI.TransformGetTransform( + r.esAPI.TransformGetTransform.WithContext(ctx), + r.esAPI.TransformGetTransform.WithTransformID(transformPattern), ) if err != nil { return "", err @@ -1880,9 +1937,9 @@ func (r *runner) getTransformId(ctx context.Context, transformPattern string) (s } func (r *runner) previewTransform(ctx context.Context, transformId string) ([]common.MapStr, error) { - resp, err := r.options.API.TransformPreviewTransform( - r.options.API.TransformPreviewTransform.WithContext(ctx), - r.options.API.TransformPreviewTransform.WithTransformID(transformId), + resp, err := r.esAPI.TransformPreviewTransform( + r.esAPI.TransformPreviewTransform.WithContext(ctx), + r.esAPI.TransformPreviewTransform.WithTransformID(transformId), ) if err != nil { return nil, err @@ -2070,7 +2127,7 @@ func (r *runner) selectVariants(variantsFile *servicedeployer.VariantsFile) []st var variantNames []string for k := range variantsFile.Variants { - if r.options.ServiceVariant != "" && r.options.ServiceVariant != k { + if r.serviceVariant != "" && r.serviceVariant != k { continue } variantNames = append(variantNames, k) @@ -2078,13 +2135,13 @@ func (r *runner) selectVariants(variantsFile *servicedeployer.VariantsFile) []st return variantNames } -func (r *runner) generateTestResult(docs []common.MapStr, specVersion semver.Version) error { - if !r.options.GenerateTestResult { +func (r *runner) generateTestResultFile(docs []common.MapStr, specVersion semver.Version) error { + if !r.generateTestResult { return nil } - rootPath := r.options.PackageRootPath - if ds := r.options.TestFolder.DataStream; ds != "" { + rootPath := r.packageRootPath + if ds := r.testFolder.DataStream; ds != "" { rootPath = filepath.Join(rootPath, "data_stream", ds) } @@ -2127,8 +2184,8 @@ func (r *runner) checkNewAgentLogs(ctx context.Context, agent agentdeployer.Depl tr := testrunner.TestResult{ TestType: TestType, Name: fmt.Sprintf("(%s logs)", patternsContainer.containerName), - Package: r.options.TestFolder.Package, - DataStream: r.options.TestFolder.DataStream, + Package: r.testFolder.Package, + DataStream: r.testFolder.DataStream, } tr.FailureMsg = e.Error() tr.FailureDetails = e.Details @@ -2164,8 +2221,8 @@ func (r *runner) checkAgentLogs(dump []stack.DumpResult, startTesting time.Time, tr := testrunner.TestResult{ TestType: TestType, Name: fmt.Sprintf("(%s logs)", patternsContainer.containerName), - Package: r.options.TestFolder.Package, - DataStream: r.options.TestFolder.DataStream, + Package: r.testFolder.Package, + DataStream: r.testFolder.DataStream, } tr.FailureMsg = e.Error() tr.FailureDetails = e.Details diff --git a/internal/testrunner/runners/system/runner_test.go b/internal/testrunner/runners/system/runner_test.go index a2b8c2743..61fea530a 100644 --- a/internal/testrunner/runners/system/runner_test.go +++ b/internal/testrunner/runners/system/runner_test.go @@ -374,11 +374,9 @@ func TestCheckAgentLogs(t *testing.T) { } runner := runner{ - options: testrunner.TestOptions{ - TestFolder: testrunner.TestFolder{ - Package: "package", - DataStream: "datastream", - }, + testFolder: testrunner.TestFolder{ + Package: "package", + DataStream: "datastream", }, } results, err := runner.checkAgentLogs(dump, startTime, tc.errorPatterns) diff --git a/internal/testrunner/testrunner.go b/internal/testrunner/testrunner.go index c79e969e4..ebb12afbc 100644 --- a/internal/testrunner/testrunner.go +++ b/internal/testrunner/testrunner.go @@ -59,15 +59,13 @@ type TestRunner interface { String() string // Run executes the test runner. - Run(context.Context, TestOptions) ([]TestResult, error) + Run(context.Context) ([]TestResult, error) // TearDown cleans up any test runner resources. It must be called // after the test runner has finished executing. TearDown(context.Context) error } -var runners = map[TestType]TestRunner{} - // TestResult contains a single test's results type TestResult struct { // Name of test result. Optional. @@ -278,19 +276,9 @@ func ExtractDataStreamFromPath(fullPath, packageRootPath string) string { return dataStream } -// RegisterRunner method registers the test runner. -func RegisterRunner(runner TestRunner) { - runners[runner.Type()] = runner -} - // Run method delegates execution to the registered test runner, based on the test type. -func Run(ctx context.Context, testType TestType, options TestOptions) ([]TestResult, error) { - runner, defined := runners[testType] - if !defined { - return nil, fmt.Errorf("unregistered runner test: %s", testType) - } - - results, err := runner.Run(ctx, options) +func Run(ctx context.Context, runner TestRunner) ([]TestResult, error) { + results, err := runner.Run(ctx) tdErr := runner.TearDown(ctx) if err != nil { return nil, fmt.Errorf("could not complete test run: %w", err) @@ -301,11 +289,6 @@ func Run(ctx context.Context, testType TestType, options TestOptions) ([]TestRes return results, nil } -// TestRunners returns registered test runners. -func TestRunners() map[TestType]TestRunner { - return runners -} - // findDataStreamTestFoldersPaths can only be called for test runners that require tests to be defined // at the data stream level. func findDataStreamTestFolderPaths(packageRootPath, dataStreamGlob, testTypeGlob string) ([]string, error) {