Add securityTest to scan Terraform templates: tfsec (globocom#478)

* build(deps): bump github.com/onsi/gomega from 1.7.0 to 1.7.1 Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.7.0 to 1.7.1. - [Release notes](https://github.com/onsi/gomega/releases) - [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md) - [Commits](onsi/gomega@v1.7.0...v1.7.1) Signed-off-by: dependabot-preview[bot] <[email protected]> * build(deps): bump github.com/onsi/ginkgo from 1.10.2 to 1.10.3 Bumps [github.com/onsi/ginkgo](https://github.com/onsi/ginkgo) from 1.10.2 to 1.10.3. - [Release notes](https://github.com/onsi/ginkgo/releases) - [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md) - [Commits](onsi/ginkgo@v1.10.2...v1.10.3) Signed-off-by: dependabot-preview[bot] <[email protected]> * build(deps): bump github.com/spf13/viper from 1.4.0 to 1.5.0 Bumps [github.com/spf13/viper](https://github.com/spf13/viper) from 1.4.0 to 1.5.0. - [Release notes](https://github.com/spf13/viper/releases) - [Commits](spf13/viper@v1.4.0...v1.5.0) Signed-off-by: dependabot-preview[bot] <[email protected]> * Feature: Add struct TFSec * Feature: Add config for tfsec in config.yaml * Feature: Add TFSec in api context * Feature: Add TFSec in securitytest * Feature: Add TFSec in types * Feature: Add TFSec final references in api module * Feature: Add TFSec into types and output in client module * Feature: Add TFSec references in cli * Feature: Update vendors * Fix: Add comment in struct Location * Feature: Add dockerfile for TFSec * Fix: Fix TFSec output in client * Feature: Update shell scripts to build and push images * Fix: Update tfsec dockerfile to use version v0.19.0 * Bump: Update go.mod * Bump: Update golangci-lint to v1.27.0 * Lint: Fix lint in files using Println * Fix: Add missing calls of printSTDOUTOutputTFSec * Fix: Change type of fields StartLine and EndLine * Fix: Add jq and git into dockerfile * Fix: Print hcl results in client * Fix: Fix wrong securitytest name * Fix: Remove vendors * Fix: Remove go.mod and go.sum * Fix: Replace wrong comment * Fix: Fix wrong commmand to get tfsec version huskyci/tfsec:latest Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> Co-authored-by: Carlos Junior <[email protected]> Co-authored-by: carlosljr <[email protected]>
Lucasdfg07 · May 21, 2020 · 22e0b14 · 22e0b14
1 parent 7d63fcc
commit 22e0b14
Show file tree

Hide file tree

Showing 21 changed files with 264 additions and 16 deletions.
diff --git a/Makefile b/Makefile
@@ -177,4 +177,4 @@ test:
 	cd cli && $(GO) tool cover -func=e.out
 
 ## Builds and push securityTest containers with the latest tags
-update-containers: build-containers push-containers
+update-containers: build-containers push-containers
diff --git a/api/config.yaml b/api/config.yaml
@@ -333,3 +333,28 @@ gitleaks:
   type: Generic
   default: true
   timeOutInSeconds: 360
+
+tfsec:
+  name: tfsec
+  image: huskyci/tfsec
+  imageTag: "v0.19.0"
+  cmd: |+
+    mkdir -p ~/.ssh &&
+    echo 'GIT_PRIVATE_SSH_KEY' > ~/.ssh/huskyci_id_rsa &&
+    chmod 600 ~/.ssh/huskyci_id_rsa &&
+    echo "IdentityFile ~/.ssh/huskyci_id_rsa" >> /etc/ssh/ssh_config &&
+    echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&
+    GIT_TERMINAL_PROMPT=0 git clone -b %GIT_BRANCH% --single-branch %GIT_REPO% code --quiet 2> /tmp/errorGitCloneTFSec
+    if [ $? -eq 0 ]; then
+        ./tfsec code --format=json | grep -v "WARNING: skipped" > pre-results.json
+        cat pre-results.json | grep -v "WARNING: skipped" > results.json
+        echo "{\"warnings\":\"$(cat pre-results.json | grep "WARNING: skipped")\"}" >> warning.json
+        cat results.json warning.json | jq -s add | jq -j -M -c .
+    else
+      echo "ERROR_CLONING"
+      cat /tmp/errorGitCloneTFSec
+    fi
+  type: Language
+  language: HCL
+  default: true
+  timeOutInSeconds: 360
diff --git a/api/context/context.go b/api/context/context.go
@@ -81,6 +81,7 @@ type APIConfig struct {
 	SpotBugsSecurityTest   *types.SecurityTest
 	GitleaksSecurityTest   *types.SecurityTest
 	SafetySecurityTest     *types.SecurityTest
+	TFSecSecurityTest      *types.SecurityTest
 	DBInstance             db.Requests
 }
 
@@ -124,6 +125,7 @@ func (dF DefaultConfig) SetOnceConfig() {
 			SpotBugsSecurityTest:   dF.getSecurityTestConfig("spotbugs"),
 			GitleaksSecurityTest:   dF.getSecurityTestConfig("gitleaks"),
 			SafetySecurityTest:     dF.getSecurityTestConfig("safety"),
+			TFSecSecurityTest:      dF.getSecurityTestConfig("tfsec"),
 			DBInstance:             dF.GetDB(),
 		}
 	})

diff --git a/api/context/context_test.go b/api/context/context_test.go
@@ -427,6 +427,16 @@ var _ = Describe("Context", func() {
 						Default:          fakeCaller.expectedBoolFromConfig,
 						TimeOutInSeconds: fakeCaller.expectedIntFromConfig,
 					},
+					TFSecSecurityTest: &types.SecurityTest{
+						Name:             fakeCaller.expectedStringFromConfig,
+						Image:            fakeCaller.expectedStringFromConfig,
+						ImageTag:         fakeCaller.expectedStringFromConfig,
+						Cmd:              fakeCaller.expectedStringFromConfig,
+						Type:             fakeCaller.expectedStringFromConfig,
+						Language:         fakeCaller.expectedStringFromConfig,
+						Default:          fakeCaller.expectedBoolFromConfig,
+						TimeOutInSeconds: fakeCaller.expectedIntFromConfig,
+					},
 					DBInstance: &db.MongoRequests{},
 				}
 				Expect(apiConfig).To(Equal(expectedConfig))

diff --git a/api/log/messagecodes.go b/api/log/messagecodes.go
@@ -73,6 +73,7 @@ var MsgCode = map[int]string{
 	1037: "Internal error running Yarnaudit: ",
 	1038: "Could not Unmarshall the following gitleaksOutput: ",
 	1039: "Could not Unmarshall the following spotbugsOutput: ",
+	1040: "Could not Unmarshall the following tfsecOutput: ",
 
 	// MongoDB infos
 	21: "Connecting to MongoDB.",

diff --git a/api/securitytest/run.go b/api/securitytest/run.go
@@ -28,6 +28,7 @@ const npmaudit = "npmaudit"
 const yarnaudit = "yarnaudit"
 const spotbugs = "spotbugs"
 const gitleaks = "gitleaks"
+const tfsec = "tfsec"
 
 // Start runs both generic and language security
 func (results *RunAllInfo) Start(enryScan SecTestScanInfo) error {
@@ -221,6 +222,8 @@ func (results *RunAllInfo) setVulns(securityTestScan SecTestScanInfo) {
 			results.HuskyCIResults.JavaResults.HuskyCISpotBugsOutput.HighVulns = append(results.HuskyCIResults.JavaResults.HuskyCISpotBugsOutput.HighVulns, highVuln)
 		case gitleaks:
 			results.HuskyCIResults.GenericResults.HuskyCIGitleaksOutput.HighVulns = append(results.HuskyCIResults.GenericResults.HuskyCIGitleaksOutput.HighVulns, highVuln)
+		case tfsec:
+			results.HuskyCIResults.HclResults.HuskyCITFSecOutput.HighVulns = append(results.HuskyCIResults.HclResults.HuskyCITFSecOutput.HighVulns, highVuln)
 		}
 	}
 
@@ -242,6 +245,8 @@ func (results *RunAllInfo) setVulns(securityTestScan SecTestScanInfo) {
 			results.HuskyCIResults.JavaResults.HuskyCISpotBugsOutput.MediumVulns = append(results.HuskyCIResults.JavaResults.HuskyCISpotBugsOutput.MediumVulns, mediumVuln)
 		case gitleaks:
 			results.HuskyCIResults.GenericResults.HuskyCIGitleaksOutput.MediumVulns = append(results.HuskyCIResults.GenericResults.HuskyCIGitleaksOutput.MediumVulns, mediumVuln)
+		case tfsec:
+			results.HuskyCIResults.HclResults.HuskyCITFSecOutput.MediumVulns = append(results.HuskyCIResults.HclResults.HuskyCITFSecOutput.MediumVulns, mediumVuln)
 		}
 	}
 
@@ -263,6 +268,8 @@ func (results *RunAllInfo) setVulns(securityTestScan SecTestScanInfo) {
 			results.HuskyCIResults.JavaResults.HuskyCISpotBugsOutput.LowVulns = append(results.HuskyCIResults.JavaResults.HuskyCISpotBugsOutput.LowVulns, lowVuln)
 		case gitleaks:
 			results.HuskyCIResults.GenericResults.HuskyCIGitleaksOutput.LowVulns = append(results.HuskyCIResults.GenericResults.HuskyCIGitleaksOutput.LowVulns, lowVuln)
+		case tfsec:
+			results.HuskyCIResults.HclResults.HuskyCITFSecOutput.LowVulns = append(results.HuskyCIResults.HclResults.HuskyCITFSecOutput.LowVulns, lowVuln)
 		}
 	}
 
@@ -284,6 +291,8 @@ func (results *RunAllInfo) setVulns(securityTestScan SecTestScanInfo) {
 			results.HuskyCIResults.JavaResults.HuskyCISpotBugsOutput.NoSecVulns = append(results.HuskyCIResults.JavaResults.HuskyCISpotBugsOutput.NoSecVulns, noSec)
 		case gitleaks:
 			results.HuskyCIResults.GenericResults.HuskyCIGitleaksOutput.NoSecVulns = append(results.HuskyCIResults.GenericResults.HuskyCIGitleaksOutput.NoSecVulns, noSec)
+		case tfsec:
+			results.HuskyCIResults.HclResults.HuskyCITFSecOutput.NoSecVulns = append(results.HuskyCIResults.HclResults.HuskyCITFSecOutput.NoSecVulns, noSec)
 		}
 	}
 }

diff --git a/api/securitytest/securitytest.go b/api/securitytest/securitytest.go
@@ -23,6 +23,7 @@ var securityTestAnalyze = map[string]func(scanInfo *SecTestScanInfo) error{
 	"spotbugs":   analyzeSpotBugs,
 	"gitleaks":   analyseGitleaks,
 	"safety":     analyzeSafety,
+	"tfsec":      analyzeTFSec,
 }
 
 // SecTestScanInfo holds all information of securityTest scan.

diff --git a/api/securitytest/tfsec.go b/api/securitytest/tfsec.go
@@ -0,0 +1,90 @@
+// Copyright 2019 Globo.com authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package securitytest
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/globocom/huskyCI/api/log"
+	"github.com/globocom/huskyCI/api/types"
+)
+
+// TFSecOutput is the struct that holds all data from TFSec output.
+type TFSecOutput struct {
+	Warnings json.RawMessage `json:"warnings"`
+	Results  []TFSecResult   `json:"results"`
+}
+
+// TFSecResult is the struct that holds detailed information of results from TFSec output.
+type TFSecResult struct {
+	RuleID      string   `json:"rule_id"`
+	Link        string   `json:"link"`
+	Location    Location `json:"location"`
+	Description string   `json:"description"`
+	Severity    string   `json:"severity"`
+}
+
+// Location is the struct that holds detailed information of location from each result
+type Location struct {
+	Filename  string `json:"filename"`
+	StartLine int    `json:"start_line"`
+	EndLine   int    `json:"end_line"`
+}
+
+func analyzeTFSec(tfsecScan *SecTestScanInfo) error {
+
+	tfsecOutput := TFSecOutput{}
+
+	// Unmarshall rawOutput into finalOutput, that is a TFSec struct.
+	if err := json.Unmarshal([]byte(tfsecScan.Container.COutput), &tfsecOutput); err != nil {
+		log.Error("analyzeTFSec", "TFSEC", 1040, tfsecScan.Container.COutput, err)
+		tfsecScan.ErrorFound = err
+		return err
+	}
+	tfsecScan.FinalOutput = tfsecOutput
+
+	// an empty Results slice states that no Issues were found.
+	if tfsecOutput.Results == nil {
+		tfsecScan.prepareContainerAfterScan()
+		return nil
+	}
+
+	// check results and prepare all vulnerabilities found
+	tfsecScan.prepareTFSecVulns()
+	tfsecScan.prepareContainerAfterScan()
+	return nil
+}
+
+func (tfsecScan *SecTestScanInfo) prepareTFSecVulns() {
+
+	huskyCItfsecResults := types.HuskyCISecurityTestOutput{}
+	tfsecOutput := tfsecScan.FinalOutput.(TFSecOutput)
+
+	for _, result := range tfsecOutput.Results {
+		tfsecVuln := types.HuskyCIVulnerability{}
+		tfsecVuln.Language = "HCL"
+		tfsecVuln.SecurityTool = "TFSec"
+		tfsecVuln.Severity = result.Severity
+		tfsecVuln.Details = result.RuleID + " @ [" + result.Description + "]"
+		startLine := strconv.Itoa(result.Location.StartLine)
+		endLine := strconv.Itoa(result.Location.EndLine)
+		tfsecVuln.Line = startLine
+		tfsecVuln.Code = fmt.Sprintf("Code beetween Line %s and Line %s.", startLine, endLine)
+		tfsecVuln.File = result.Location.Filename
+
+		switch tfsecVuln.Severity {
+		case "INFO":
+			huskyCItfsecResults.LowVulns = append(huskyCItfsecResults.LowVulns, tfsecVuln)
+		case "WARNING":
+			huskyCItfsecResults.MediumVulns = append(huskyCItfsecResults.MediumVulns, tfsecVuln)
+		case "ERROR":
+			huskyCItfsecResults.HighVulns = append(huskyCItfsecResults.HighVulns, tfsecVuln)
+		}
+	}
+
+	tfsecScan.Vulnerabilities = huskyCItfsecResults
+}
diff --git a/api/types/types.go b/api/types/types.go
@@ -96,6 +96,7 @@ type HuskyCIResults struct {
 	JavaScriptResults JavaScriptResults `bson:"javascriptresults,omitempty" json:"javascriptresults,omitempty"`
 	RubyResults       RubyResults       `bson:"rubyresults,omitempty" json:"rubyresults,omitempty"`
 	JavaResults       JavaResults       `bson:"javaresults,omitempty" json:"javaresults,omitempty"`
+	HclResults        HclResults        `bson:"hclresults,omitempty" json:"hclresults,omitempty"`
 	GenericResults    GenericResults    `bson:"genericresults,omitempty" json:"genericresults,omitempty"`
 }
 
@@ -131,6 +132,11 @@ type GenericResults struct {
 	HuskyCIGitleaksOutput HuskyCISecurityTestOutput `bson:"gitleaksoutput,omitempty" json:"gitleaksoutput,omitempty"`
 }
 
+// HclResults represents all HCL security tests results.
+type HclResults struct {
+	HuskyCITFSecOutput HuskyCISecurityTestOutput `bson:"tfsecoutput,omitempty" json:"tfsecoutput,omitempty"`
+}
+
 // HuskyCISecurityTestOutput stores all Low, Medium and High vulnerabilities for a sec test
 type HuskyCISecurityTestOutput struct {
 	NoSecVulns  []HuskyCIVulnerability `bson:"nosecvulns,omitempty" json:"nosecvulns,omitempty"`

diff --git a/api/util/api/api.go b/api/util/api/api.go
@@ -117,7 +117,7 @@ func (cH *CheckUtils) checkDB(configAPI *apiContext.APIConfig) error {
 }
 
 func (cH *CheckUtils) checkEachSecurityTest(configAPI *apiContext.APIConfig) error {
-	securityTests := []string{"enry", "gitauthors", "gosec", "brakeman", "bandit", "npmaudit", "yarnaudit", "spotbugs", "gitleaks", "safety"}
+	securityTests := []string{"enry", "gitauthors", "gosec", "brakeman", "bandit", "npmaudit", "yarnaudit", "spotbugs", "gitleaks", "safety", "tfsec"}
 	for _, securityTest := range securityTests {
 		if err := checkSecurityTest(securityTest, configAPI); err != nil {
 			errMsg := fmt.Sprintf("%s %s", securityTest, err)
@@ -171,6 +171,8 @@ func checkSecurityTest(securityTestName string, configAPI *apiContext.APIConfig)
 		securityTestConfig = *configAPI.GitleaksSecurityTest
 	case "safety":
 		securityTestConfig = *configAPI.SafetySecurityTest
+	case "tfsec":
+		securityTestConfig = *configAPI.TFSecSecurityTest
 	default:
 		return errors.New("securityTest name not defined")
 	}

diff --git a/cli/analysis/analysis.go b/cli/analysis/analysis.go
@@ -61,7 +61,8 @@ func (a *Analysis) CheckPath(path string) error {
 	}
 
 	for language := range a.getAvailableSecurityTests(a.Languages) {
-		fmt.Println(fmt.Sprintf("[HUSKYCI] %s found.", language))
+		s := fmt.Sprintf("[HUSKYCI] %s found.", language)
+		fmt.Println(s)
 	}
 
 	return nil
@@ -174,6 +175,8 @@ func (a *Analysis) getAvailableSecurityTests(languages []string) map[string][]st
 			list[language] = []string{"huskyci/npmaudit", "huskyci/yarnaudit"}
 		case "Java":
 			list[language] = []string{"huskyci/spotbugs"}
+		case "HCL":
+			list[language] = []string{"huskyci/tfsec"}
 		}
 	}
 

diff --git a/cli/cmd/root.go b/cli/cmd/root.go
@@ -24,8 +24,8 @@ var (
 huskyCI is an open source tool that orchestrates security tests and
 centralizes all results into a database for further analysis and metrics.
 It can perform static security analysis in Python (Bandit and Safety),
-Ruby (Brakeman), JavaScript (Npm Audit and Yarn Audit), Golang (Gosec),
-and Java (SpotBugs plus Find Sec Bugs). It can also audit repositories
+Ruby (Brakeman), JavaScript (Npm Audit and Yarn Audit), Golang (Gosec), 
+Java (SpotBugs plus Find Sec Bugs) and HCL (TFSec). It can also audit repositories
 for secrets like AWS Secret Keys, Private SSH Keys, and many others using
 GitLeaks.
 		`,

diff --git a/client/analysis/analysis.go b/client/analysis/analysis.go
@@ -65,7 +65,7 @@ func StartAnalysis() (string, error) {
 
 	RID := resp.Header.Get("X-Request-Id")
 	if RID == "" {
-		errorMsg := fmt.Sprintf("Error sending request to start analysis. RID is empty!")
+		errorMsg := "Error sending request to start analysis. RID is empty!"
 		return "", errors.New(errorMsg)
 	}