Merge branch 'master' into contrast-step

cmd/checkIfStepActive.go

@@ -6,12 +6,12 @@ import (
 	"io"
 	"os"
 
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
 	"github.com/SAP/jenkins-library/pkg/config"
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/piperutils"
-	"github.com/bmatcuk/doublestar"
-	"github.com/pkg/errors"
-	"github.com/spf13/cobra"
 )
 
 type checkStepActiveCommandOptions struct {
@@ -63,6 +63,9 @@ func checkIfStepActive(utils piperutils.FileUtils) error {
 	if checkStepActiveOptions.stageName == "" {
 		return errors.New("stage name must not be empty")
 	}
+	if checkStepActiveOptions.v1Active {
+		log.Entry().Warning("Please do not use --useV1 flag since it is deprecated and will be removed in future releases")
+	}
 	var pConfig config.Config
 
 	// load project config and defaults
@@ -78,31 +81,15 @@ func checkIfStepActive(utils piperutils.FileUtils) error {
 	}
 	defer stageConfigFile.Close()
 
-	var runSteps map[string]map[string]bool
-	var runStages map[string]bool
-
 	// load and evaluate step conditions
-	if checkStepActiveOptions.v1Active {
-		runConfig := config.RunConfig{StageConfigFile: stageConfigFile}
-		runConfigV1 := &config.RunConfigV1{RunConfig: runConfig}
-		err = runConfigV1.InitRunConfigV1(projectConfig, utils, GeneralConfig.EnvRootPath)
-		if err != nil {
-			return err
-		}
-		runSteps = runConfigV1.RunSteps
-		runStages = runConfigV1.RunStages
-	} else {
-		log.Entry().Warning("This step is using deprecated format of stage conditions which will be removed in Jan 2024. " +
-			"To avoid pipeline breakage, please call checkIfStepActive command with --useV1 flag.",
-		)
-		runConfig := &config.RunConfig{StageConfigFile: stageConfigFile}
-		err = runConfig.InitRunConfig(projectConfig, nil, nil, nil, nil, doublestar.Glob, checkStepActiveOptions.openFile)
-		if err != nil {
-			return err
-		}
-		runSteps = runConfig.RunSteps
-		runStages = runConfig.RunStages
+	runConfig := config.RunConfig{StageConfigFile: stageConfigFile}
+	runConfigV1 := &config.RunConfigV1{RunConfig: runConfig}
+	err = runConfigV1.InitRunConfigV1(projectConfig, utils, GeneralConfig.EnvRootPath)
+	if err != nil {
+		return err
 	}
+	runSteps := runConfigV1.RunSteps
+	runStages := runConfigV1.RunStages
 
 	log.Entry().Debugf("RunSteps: %v", runSteps)
 	log.Entry().Debugf("RunStages: %v", runStages)
@@ -149,7 +136,7 @@ func addCheckStepActiveFlags(cmd *cobra.Command) {
 		"Default config of piper pipeline stages")
 	cmd.Flags().StringVar(&checkStepActiveOptions.stepName, "step", "", "Name of the step being checked")
 	cmd.Flags().StringVar(&checkStepActiveOptions.stageName, "stage", "", "Name of the stage in which contains the step being checked")
-	cmd.Flags().BoolVar(&checkStepActiveOptions.v1Active, "useV1", false, "Use new CRD-style stage configuration")
+	cmd.Flags().BoolVar(&checkStepActiveOptions.v1Active, "useV1", false, "Use new CRD-style stage configuration (deprecated)")
 	cmd.Flags().StringVar(&checkStepActiveOptions.stageOutputFile, "stageOutputFile", "", "Defines a file path. If set, the stage output will be written to the defined file")
 	cmd.Flags().StringVar(&checkStepActiveOptions.stepOutputFile, "stepOutputFile", "", "Defines a file path. If set, the step output will be written to the defined file")
 	_ = cmd.MarkFlagRequired("step")

cmd/checkIfStepActive_test.go

@@ -25,11 +25,14 @@ stages:
 steps:`
 	case "stage-config.yml":
 		fileContent = `
-stages:
-  testStage:
-    stepConditions:
-      testStep:
-        config: testConfig`
+spec:
+  stages:
+    - name: testStage
+      displayName: testStage
+      steps:
+        - name: testStep
+          conditions:
+            - configKey: testConfig`
 	case ".pipeline/config.yml":
 		fileContent = `
 steps: 

cmd/detectExecuteScan.go

@@ -20,6 +20,7 @@ import (
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/maven"
 	"github.com/SAP/jenkins-library/pkg/orchestrator"
+	"github.com/SAP/jenkins-library/pkg/piperenv"
 	"github.com/SAP/jenkins-library/pkg/piperutils"
 	"github.com/SAP/jenkins-library/pkg/reporting"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
@@ -30,6 +31,8 @@ import (
 	"github.com/pkg/errors"
 )
 
+const NO_VERSION_SUFFIX = ""
+
 type detectUtils interface {
 	piperutils.FileUtils
 
@@ -202,7 +205,7 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec
 	blackduckSystem := newBlackduckSystem(config)
 
 	args := []string{"./detect.sh"}
-	args, err = addDetectArgs(args, config, utils, blackduckSystem)
+	args, err = addDetectArgs(args, config, utils, blackduckSystem, NO_VERSION_SUFFIX, NO_VERSION_SUFFIX)
 	if err != nil {
 		return err
 	}
@@ -214,7 +217,18 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec
 	utils.SetDir(".")
 	utils.SetEnv(envs)
 
-	err = utils.RunShell("/bin/bash", script)
+	err = mapDetectError(utils.RunShell("/bin/bash", script), config, utils)
+	if config.ScanContainerDistro != "" {
+		imageError := mapDetectError(runDetectImages(ctx, config, utils, blackduckSystem, influx, blackduckSystem), config, utils)
+		if imageError != nil {
+			if err != nil {
+				err = errors.Wrapf(err, "error during scanning images: %q", imageError.Error())
+			} else {
+				err = imageError
+			}
+		}
+	}
+
 	reportingErr := postScanChecksAndReporting(ctx, config, influx, utils, blackduckSystem)
 	if reportingErr != nil {
 		if strings.Contains(reportingErr.Error(), "License Policy Violations found") {
@@ -227,6 +241,17 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec
 			log.Entry().Warnf("Failed to generate reports: %v", reportingErr)
 		}
 	}
+	// create Toolrecord file
+	toolRecordFileName, toolRecordErr := createToolRecordDetect(utils, "./", config, blackduckSystem)
+	if toolRecordErr != nil {
+		// do not fail until the framework is well established
+		log.Entry().Warning("TR_DETECT: Failed to create toolrecord file "+toolRecordFileName, err)
+	}
+
+	return err
+}
+
+func mapDetectError(err error, config detectExecuteScanOptions, utils detectUtils) error {
 	if err != nil {
 		// Setting error category based on exit code
 		mapErrorCategory(utils.GetExitCode())
@@ -238,15 +263,52 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec
 			err = errors.Wrapf(err, exitCodeMapping(utils.GetExitCode()))
 		}
 	}
-	// create Toolrecord file
-	toolRecordFileName, toolRecordErr := createToolRecordDetect(utils, "./", config, blackduckSystem)
-	if toolRecordErr != nil {
-		// do not fail until the framework is well established
-		log.Entry().Warning("TR_DETECT: Failed to create toolrecord file "+toolRecordFileName, err)
-	}
 	return err
 }
 
+func runDetectImages(ctx context.Context, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem, influx *detectExecuteScanInflux, blackduckSystem *blackduckSystem) error {
+	cpePath := filepath.Join(GeneralConfig.EnvRootPath, "commonPipelineEnvironment")
+	imagesRaw := piperenv.GetResourceParameter(cpePath, "container", "imageNameTags.json")
+	if imagesRaw == "" {
+		log.Entry().Debugf("No images found to be scanned")
+		return nil
+	}
+
+	var err error
+	log.Entry().Infof("Scanning %d images", len(config.ImageNameTags))
+	for _, image := range config.ImageNameTags {
+		// Download image to be scanned
+		log.Entry().Debugf("Scanning image: %q", image)
+		tarName := fmt.Sprintf("%s.tar", strings.Split(image, ":")[0])
+
+		options := containerSaveImageOptions{
+			ContainerRegistryURL:      config.RegistryURL,
+			ContainerImage:            image,
+			ContainerRegistryPassword: config.RepositoryPassword,
+			ContainerRegistryUser:     config.RepositoryUsername,
+			FilePath:                  tarName,
+			ImageFormat:               "legacy",
+		}
+		containerSaveImage(options, &telemetry.CustomData{})
+
+		args := []string{"./detect.sh"}
+		args, err = addDetectArgsImages(args, config, utils, sys, tarName)
+		if err != nil {
+			return err
+		}
+		script := strings.Join(args, " ")
+
+		err = utils.RunShell("/bin/bash", script)
+		err = mapDetectError(err, config, utils)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // Get proper error category
 func mapErrorCategory(exitCodeKey int) {
 	switch exitCodeKey {
@@ -331,8 +393,11 @@ func getDetectScript(config detectExecuteScanOptions, utils detectUtils) error {
 	return nil
 }
 
-func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem) ([]string, error) {
+func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem, versionSuffix, locationSuffix string) ([]string, error) {
 	detectVersionName := getVersionName(config)
+	if versionSuffix != NO_VERSION_SUFFIX {
+		detectVersionName = fmt.Sprintf("%s-%s", detectVersionName, versionSuffix)
+	}
 	// Split on spaces, the scanPropeties, so that each property is available as a single string
 	// instead of all properties being part of a single string
 	config.ScanProperties = piperutils.SplitAndTrim(config.ScanProperties, " ")
@@ -393,6 +458,10 @@ func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectU
 	codelocation := config.CodeLocation
 	if len(codelocation) == 0 && len(config.ProjectName) > 0 {
 		codelocation = fmt.Sprintf("%v/%v", config.ProjectName, detectVersionName)
+
+		if locationSuffix != "" {
+			codelocation = fmt.Sprintf("%v-%v", codelocation, locationSuffix)
+		}
 	}
 
 	args = append(args, fmt.Sprintf("\"--detect.project.name=%v\"", config.ProjectName))
@@ -467,6 +536,39 @@ func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectU
 	return args, nil
 }
 
+func addDetectArgsImages(args []string, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem, imageTar string) ([]string, error) {
+	// suffix := strings.Split(imageTar, ".")[0]
+	// In order to preserve source scan result
+	config.Unmap = false
+	args, err := addDetectArgs(args, config, utils, sys, NO_VERSION_SUFFIX, fmt.Sprintf("image-%s", strings.Split(imageTar, ".")[0]))
+	if err != nil {
+		return []string{}, err
+	}
+
+	args = append(args, fmt.Sprintf("--detect.docker.tar=./%s", imageTar))
+	args = append(args, "--detect.target.type=IMAGE")
+	// https://community.synopsys.com/s/article/Docker-image-scanning-CLI-examples-and-some-Q-As
+	args = append(args, "--detect.tools.excluded=DETECTOR")
+	args = append(args, "--detect.docker.passthrough.shared.dir.path.local=/opt/blackduck/blackduck-imageinspector/shared/")
+	args = append(args, "--detect.docker.passthrough.shared.dir.path.imageinspector=/opt/blackduck/blackduck-imageinspector/shared")
+	args = append(args, fmt.Sprintf("--detect.docker.passthrough.imageinspector.service.distro.default=%s", config.ScanContainerDistro))
+	args = append(args, "--detect.docker.passthrough.imageinspector.service.start=false")
+	args = append(args, "--detect.docker.passthrough.output.include.squashedimage=false")
+
+	switch config.ScanContainerDistro {
+	case "ubuntu":
+		args = append(args, "--detect.docker.passthrough.imageinspector.service.url=http://localhost:8082")
+	case "centos":
+		args = append(args, "--detect.docker.passthrough.imageinspector.service.url=http://localhost:8081")
+	case "alpine":
+		args = append(args, "--detect.docker.passthrough.imageinspector.service.url=http://localhost:8080")
+	default:
+		return nil, fmt.Errorf("unknown container distro %q", config.ScanContainerDistro)
+	}
+
+	return args, nil
+}
+
 func getVersionName(config detectExecuteScanOptions) string {
 	detectVersionName := config.CustomScanVersion
 	if len(detectVersionName) > 0 {

cmd/detectExecuteScan_test.go

@@ -807,7 +807,7 @@ func TestAddDetectArgs(t *testing.T) {
 			config := detectExecuteScanOptions{Token: "token", ServerURL: "https://my.blackduck.system", ProjectName: v.options.ProjectName, Version: v.options.Version, CustomScanVersion: v.options.CustomScanVersion}
 			sys := newBlackduckMockSystem(config)
 
-			got, err := addDetectArgs(v.args, v.options, newDetectTestUtilsBundle(v.isPullRequest), &sys)
+			got, err := addDetectArgs(v.args, v.options, newDetectTestUtilsBundle(v.isPullRequest), &sys, NO_VERSION_SUFFIX, "")
 			assert.NoError(t, err)
 			assert.Equal(t, v.expected, got)
 		})