From bc5041a381539522c2f615c80df37bc5281904ce Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Thu, 16 Jan 2025 15:02:45 +0000 Subject: [PATCH] base logic for line numbers, still wrong as the gitlab ui strangely counts newlines --- src/pipeleak/helper/helper.go | 9 +++++++++ src/pipeleak/scanner/queue.go | 2 +- src/pipeleak/scanner/rules.go | 30 ++++++++++++++++++++++++++++-- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/pipeleak/helper/helper.go b/src/pipeleak/helper/helper.go index 272a010..ff00b58 100644 --- a/src/pipeleak/helper/helper.go +++ b/src/pipeleak/helper/helper.go @@ -11,6 +11,7 @@ import ( "os/signal" "path" "regexp" + "strconv" "strings" "syscall" "time" @@ -220,3 +221,11 @@ func IsDirectory(path string) bool { return fileInfo.IsDir() } + +func LineNRToWebURL(webUrl string, lineNumber int) string { + if (lineNumber < 0) { + return webUrl + } + + return webUrl + "#L" + strconv.Itoa(lineNumber) +} \ No newline at end of file diff --git a/src/pipeleak/scanner/queue.go b/src/pipeleak/scanner/queue.go index ced66b9..376ecc3 100644 --- a/src/pipeleak/scanner/queue.go +++ b/src/pipeleak/scanner/queue.go @@ -104,7 +104,7 @@ func analyzeJobTrace(git *gitlab.Client, item QueueItem, options *ScanOptions) { findings := DetectHits(trace, options.MaxScanGoRoutines, options.TruffleHogVerification) for _, finding := range findings { - log.Warn().Str("confidence", finding.Pattern.Pattern.Confidence).Str("ruleName", finding.Pattern.Pattern.Name).Str("value", finding.Text).Str("url", item.Meta.JobWebUrl).Str("jobName", item.Meta.JobName).Msg("HIT") + log.Warn().Str("confidence", finding.Pattern.Pattern.Confidence).Str("ruleName", finding.Pattern.Pattern.Name).Str("value", finding.Text).Str("url", helper.LineNRToWebURL(item.Meta.JobWebUrl, finding.LineNumber)).Str("jobName", item.Meta.JobName).Msg("HIT") } } diff --git a/src/pipeleak/scanner/rules.go b/src/pipeleak/scanner/rules.go index 27ce204..fd3ac1f 100644 --- a/src/pipeleak/scanner/rules.go +++ b/src/pipeleak/scanner/rules.go @@ -41,6 +41,7 @@ type PatternPattern struct { type Finding struct { Pattern PatternElement Text string + LineNumber int } // hold patterns in memory during runtime @@ -152,6 +153,7 @@ func DetectHits(text []byte, maxThreads int, enableTruffleHogVerification bool) hits := m.FindAllIndex(text, -1) for _, hit := range hits { + lineNumber := countNewlinesUntilIndex(text, hit[0]) // truncate output to max 1024 chars for output readability hitStr := extractHitWithSurroundingText(text, hit, 50) hitStr = cleanHitLine(hitStr) @@ -160,7 +162,7 @@ func DetectHits(text []byte, maxThreads int, enableTruffleHogVerification bool) } if hitStr != "" { - findingsYml = append(findingsYml, Finding{Pattern: pattern, Text: hitStr}) + findingsYml = append(findingsYml, Finding{Pattern: pattern, Text: hitStr, LineNumber: lineNumber}) } } @@ -190,7 +192,9 @@ func DetectHits(text []byte, maxThreads int, enableTruffleHogVerification bool) if len(result.RawV2) > 0 { secret = result.RawV2 } - finding := Finding{Pattern: PatternElement{Pattern: PatternPattern{Name: result.DetectorType.String(), Confidence: "high-verified"}}, Text: string(secret)} + resultIndex := strings.Index(string(text), string(secret)) + lineNumber := countNewlinesUntilIndex(text, resultIndex) + finding := Finding{Pattern: PatternElement{Pattern: PatternPattern{Name: result.DetectorType.String(), Confidence: "high-verified"}}, Text: string(secret), LineNumber: lineNumber} // if trufflehog verification is enalbed ONLY verified rules are reported if result.Verified { @@ -277,3 +281,25 @@ func cleanHitLine(text string) string { text = strings.ReplaceAll(text, "\n", " ") return stripansi.Strip(text) } + +func countNewlinesUntilIndex(s []byte, index int) int { + if index > len(s) { + index = len(s) + } + if index == 0 { + return 0 + } + if index < 0 { + return -1 + } + substring := s[:index] + + + count := 0 + for i := 0; i < len(substring); i++ { + if substring[i] == '[' { + count++ + } + } + return count +}