diff --git a/core/config.go b/core/config.go index 9170835..3eb219f 100644 --- a/core/config.go +++ b/core/config.go @@ -38,7 +38,7 @@ func ParseConfig(options *Options) (*Config, error) { ) if len(*options.ConfigPath) > 0 { - data, err = ioutil.ReadFile(path.Join(*options.ConfigPath, "config.yaml")) + data, err = ioutil.ReadFile(path.Join(*options.ConfigPath, *options.ConfigName)) if err != nil { return config, err } @@ -47,10 +47,10 @@ func ParseConfig(options *Options) (*Config, error) { // Helps e.g. with Drone where workdir is different than shhgit dir ex, err := os.Executable() dir := filepath.Dir(ex) - data, err = ioutil.ReadFile(path.Join(dir, "config.yaml")) + data, err = ioutil.ReadFile(path.Join(dir, *options.ConfigName)) if err != nil { dir, _ = os.Getwd() - data, err = ioutil.ReadFile(path.Join(dir, "config.yaml")) + data, err = ioutil.ReadFile(path.Join(dir, *options.ConfigName)) if err != nil { return config, err } diff --git a/core/log.go b/core/log.go index 4e38f31..0bbe5c6 100644 --- a/core/log.go +++ b/core/log.go @@ -30,6 +30,7 @@ var LogColors = map[int]*color.Color{ type Logger struct { sync.Mutex + s *Session debug bool silent bool @@ -61,10 +62,10 @@ func (l *Logger) Log(level int, format string, args ...interface{}) { fmt.Printf("\r"+format+"\n", args...) } - if level > WARN && session.Config.Webhook != "" { + if level > WARN && l.s.Config.Webhook != "" { text := colorStrip(fmt.Sprintf(format, args...)) - payload := fmt.Sprintf(session.Config.WebhookPayload, text) - http.Post(session.Config.Webhook, "application/json", strings.NewReader(payload)) + payload := fmt.Sprintf(l.s.Config.WebhookPayload, text) + http.Post(l.s.Config.Webhook, "application/json", strings.NewReader(payload)) } if level == FATAL { diff --git a/core/match.go b/core/match.go index 9be78e6..9d6f2b6 100644 --- a/core/match.go +++ b/core/match.go @@ -28,16 +28,16 @@ func NewMatchFile(path string) MatchFile { } } -func IsSkippableFile(path string) bool { +func IsSkippableFile(s *Session, path string) bool { extension := strings.ToLower(filepath.Ext(path)) - for _, skippableExt := range session.Config.BlacklistedExtensions { + for _, skippableExt := range s.Config.BlacklistedExtensions { if extension == skippableExt { return true } } - for _, skippablePathIndicator := range session.Config.BlacklistedPaths { + for _, skippablePathIndicator := range s.Config.BlacklistedPaths { skippablePathIndicator = strings.Replace(skippablePathIndicator, "{sep}", string(os.PathSeparator), -1) if strings.Contains(path, skippablePathIndicator) { return true @@ -47,12 +47,12 @@ func IsSkippableFile(path string) bool { return false } -func (match MatchFile) CanCheckEntropy() bool { +func (match MatchFile) CanCheckEntropy(s *Session) bool { if match.Filename == "id_rsa" { return false } - for _, skippableExt := range session.Config.BlacklistedEntropyExtensions { + for _, skippableExt := range s.Config.BlacklistedEntropyExtensions { if match.Extension == skippableExt { return false } @@ -61,12 +61,12 @@ func (match MatchFile) CanCheckEntropy() bool { return true } -func GetMatchingFiles(dir string) []MatchFile { +func GetMatchingFiles(s *Session, dir string) []MatchFile { fileList := make([]MatchFile, 0) - maxFileSize := *session.Options.MaximumFileSize * 1024 + maxFileSize := *s.Options.MaximumFileSize * 1024 filepath.Walk(dir, func(path string, f os.FileInfo, err error) error { - if err != nil || f.IsDir() || uint(f.Size()) > maxFileSize || IsSkippableFile(path) { + if err != nil || f.IsDir() || uint(f.Size()) > maxFileSize || IsSkippableFile(s, path) { return nil } fileList = append(fileList, NewMatchFile(path)) diff --git a/core/options.go b/core/options.go index ebf2a96..bf3354f 100644 --- a/core/options.go +++ b/core/options.go @@ -1,7 +1,6 @@ package core import ( - "flag" "os" "path/filepath" ) @@ -18,34 +17,95 @@ type Options struct { PathChecks *bool ProcessGists *bool TempDirectory *string - CsvPath *string + CSVPath *string SearchQuery *string Local *string Live *string ConfigPath *string + ConfigName *string } -func ParseOptions() (*Options, error) { - options := &Options{ - Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), - Silent: flag.Bool("silent", false, "Suppress all output except for errors"), - Debug: flag.Bool("debug", false, "Print debugging information"), - MaximumRepositorySize: flag.Uint("maximum-repository-size", 5120, "Maximum repository size to process in KB"), - MaximumFileSize: flag.Uint("maximum-file-size", 256, "Maximum file size to process in KB"), - CloneRepositoryTimeout: flag.Uint("clone-repository-timeout", 10, "Maximum time it should take to clone a repository in seconds. Increase this if you have a slower connection"), - EntropyThreshold: flag.Float64("entropy-threshold", 5.0, "Set to 0 to disable entropy checks"), - MinimumStars: flag.Uint("minimum-stars", 0, "Only process repositories with this many stars. Default 0 will ignore star count"), - PathChecks: flag.Bool("path-checks", true, "Set to false to disable checking of filepaths, i.e. just match regex patterns of file contents"), - ProcessGists: flag.Bool("process-gists", true, "Will watch and process Gists. Set to false to disable."), - TempDirectory: flag.String("temp-directory", filepath.Join(os.TempDir(), Name), "Directory to process and store repositories/matches"), - CsvPath: flag.String("csv-path", "", "CSV file path to log found secrets to. Leave blank to disable"), - SearchQuery: flag.String("search-query", "", "Specify a search string to ignore signatures and filter on files containing this string (regex compatible)"), - Local: flag.String("local", "", "Specify local directory (absolute path) which to scan. Scans only given directory recursively. No need to have GitHub tokens with local run."), - Live: flag.String("live", "", "Your shhgit live endpoint"), - ConfigPath: flag.String("config-path", "", "Searches for config.yaml from given directory. If not set, tries to find if from shhgit binary's and current directory"), +func (o *Options) Merge(new *Options) { + if new.Threads != nil { + o.Threads = new.Threads } + if new.Silent != nil { + o.Silent = new.Silent + } + if new.Debug != nil { + o.Debug = new.Debug + } + if new.MaximumRepositorySize != nil { + o.MaximumRepositorySize = new.MaximumRepositorySize + } + if new.MaximumFileSize != nil { + o.MaximumFileSize = new.MaximumFileSize + } + if new.CloneRepositoryTimeout != nil { + o.CloneRepositoryTimeout = new.CloneRepositoryTimeout + } + if new.EntropyThreshold != nil { + o.EntropyThreshold = new.EntropyThreshold + } + if new.MinimumStars != nil { + o.MinimumStars = new.MinimumStars + } + if new.PathChecks != nil { + o.PathChecks = new.PathChecks + } + if new.ProcessGists != nil { + o.ProcessGists = new.ProcessGists + } + if new.TempDirectory != nil { + o.TempDirectory = new.TempDirectory + } + if new.CSVPath != nil { + o.CSVPath = new.CSVPath + } + if new.SearchQuery != nil { + o.SearchQuery = new.SearchQuery + } + if new.Local != nil { + o.Local = new.Local + } + if new.Live != nil { + o.Live = new.Live + } + if new.ConfigPath != nil { + o.ConfigPath = new.ConfigPath + } + if new.ConfigName != nil { + o.ConfigName = new.ConfigName + } +} - flag.Parse() +var ( + // Defaults that don't represent Go struct defaults + DefaultMaximumRepositorySize = uint(5120) + DefaultMaximumFileSize = uint(256) + DefaultCloneRepositoryTimeout = uint(10) + DefaultEntropy = float64(5.0) + DefaultPathChecks = true + DefaultProcessGists = true + DefaultSilent = false + DefaultMinimumStars = uint(0) + DefaultDebug = false + DefaultTempDirectory = filepath.Join(os.TempDir(), Name) + DefaultThreads = 0 + DefaultCSVPath = "" - return options, nil -} + DefaultOptions = Options{ + MaximumRepositorySize: &DefaultMaximumRepositorySize, + MaximumFileSize: &DefaultMaximumFileSize, + CloneRepositoryTimeout: &DefaultCloneRepositoryTimeout, + EntropyThreshold: &DefaultEntropy, + PathChecks: &DefaultPathChecks, + ProcessGists: &DefaultProcessGists, + TempDirectory: &DefaultTempDirectory, + Silent: &DefaultSilent, + Debug: &DefaultDebug, + Threads: &DefaultThreads, + MinimumStars: &DefaultMinimumStars, + CSVPath: &DefaultCSVPath, + } +) diff --git a/core/session.go b/core/session.go index 12f3dcf..02103d1 100644 --- a/core/session.go +++ b/core/session.go @@ -31,12 +31,6 @@ type Session struct { CsvWriter *csv.Writer } -var ( - session *Session - sessionSync sync.Once - err error -) - func (s *Session) Start() { rand.Seed(time.Now().Unix()) @@ -48,7 +42,7 @@ func (s *Session) Start() { } func (s *Session) InitLogger() { - s.Log = &Logger{} + s.Log = &Logger{s: s} s.Log.SetDebug(*s.Options.Debug) s.Log.SetSilent(*s.Options.Silent) } @@ -131,17 +125,19 @@ func (s *Session) InitThreads() { } func (s *Session) InitCsvWriter() { - if *s.Options.CsvPath == "" { + if *s.Options.CSVPath == "" { return } writeHeader := false - if !PathExists(*s.Options.CsvPath) { + if !PathExists(*s.Options.CSVPath) { writeHeader = true } - file, err := os.OpenFile(*s.Options.CsvPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - LogIfError("Could not create/open CSV file", err) + file, err := os.OpenFile(*s.Options.CSVPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + s.Log.Error("Could not create/open CSV file: %v", err) + } s.CsvWriter = csv.NewWriter(file) @@ -151,7 +147,7 @@ func (s *Session) InitCsvWriter() { } func (s *Session) WriteToCsv(line []string) { - if *s.Options.CsvPath == "" { + if *s.Options.CSVPath == "" { return } @@ -159,27 +155,22 @@ func (s *Session) WriteToCsv(line []string) { s.CsvWriter.Flush() } -func GetSession() *Session { - sessionSync.Do(func() { - session = &Session{ - Context: context.Background(), - Repositories: make(chan GitResource, 1000), - Gists: make(chan string, 100), - Comments: make(chan string, 1000), - } - - if session.Options, err = ParseOptions(); err != nil { - fmt.Println(err) - os.Exit(1) - } - - if session.Config, err = ParseConfig(session.Options); err != nil { - fmt.Println(err) - os.Exit(1) - } - - session.Start() - }) +func NewSession(ctx context.Context, o *Options) (*Session, error) { + s := &Session{ + // TODO: Remove (contexts should not be embedded in structs) + Context: ctx, + Repositories: make(chan GitResource, 1000), + Gists: make(chan string, 100), + Comments: make(chan string, 1000), + Options: &DefaultOptions, + } - return session + s.Options.Merge(o) + sc, err := ParseConfig(s.Options) + if err != nil { + return nil, fmt.Errorf("parse config: %w", err) + } + s.Config = sc + s.Start() + return s, nil } diff --git a/core/signatures.go b/core/signatures.go index e612868..115bacc 100644 --- a/core/signatures.go +++ b/core/signatures.go @@ -18,8 +18,9 @@ const ( type Signature interface { Name() string - Match(file MatchFile) (bool, string) - GetContentsMatches(contents []byte) []string + Match(MatchFile) (bool, string) + GetContentsMatches(*Session, []byte) []string + StringSubMatches(*Session, []byte) [][]string } type SimpleSignature struct { @@ -57,7 +58,11 @@ func (s SimpleSignature) Match(file MatchFile) (bool, string) { return (s.match == *haystack), matchPart } -func (s SimpleSignature) GetContentsMatches(contents []byte) []string { +func (s SimpleSignature) GetContentsMatches(sess *Session, contents []byte) []string { + return nil +} + +func (s SimpleSignature) StringSubMatches(sess *Session, contents []byte) [][]string { return nil } @@ -90,14 +95,14 @@ func (s PatternSignature) Match(file MatchFile) (bool, string) { return s.match.MatchString(*haystack), matchPart } -func (s PatternSignature) GetContentsMatches(contents []byte) []string { +func (s PatternSignature) GetContentsMatches(sess *Session, contents []byte) []string { matches := make([]string, 0) for _, match := range s.match.FindAllSubmatch(contents, -1) { match := string(match[0]) blacklistedMatch := false - for _, blacklistedString := range session.Config.BlacklistedStrings { + for _, blacklistedString := range sess.Config.BlacklistedStrings { if strings.Contains(strings.ToLower(match), strings.ToLower(blacklistedString)) { blacklistedMatch = true } @@ -111,6 +116,27 @@ func (s PatternSignature) GetContentsMatches(contents []byte) []string { return matches } +func (s PatternSignature) StringSubMatches(sess *Session, contents []byte) [][]string { + matches := [][]string{} + + for _, match := range s.match.FindAllStringSubmatch(string(contents), -1) { + fullMatch := string(match[0]) + blacklistedMatch := false + + for _, blacklistedString := range sess.Config.BlacklistedStrings { + if strings.Contains(strings.ToLower(fullMatch), strings.ToLower(blacklistedString)) { + blacklistedMatch = true + } + } + + if !blacklistedMatch { + matches = append(matches, match) + } + } + + return matches +} + func (s PatternSignature) Name() string { return s.name } diff --git a/core/util.go b/core/util.go index ff1dfb6..2339c84 100644 --- a/core/util.go +++ b/core/util.go @@ -9,8 +9,8 @@ import ( "strings" ) -func GetTempDir(suffix string) string { - dir := filepath.Join(*session.Options.TempDirectory, suffix) +func GetTempDir(s *Session, suffix string) string { + dir := filepath.Join(*s.Options.TempDirectory, suffix) if _, err := os.Stat(dir); os.IsNotExist(err) { os.MkdirAll(dir, os.ModePerm) @@ -34,12 +34,6 @@ func PathExists(path string) bool { return false } -func LogIfError(text string, err error) { - if err != nil { - GetSession().Log.Error("%s (%s", text, err.Error()) - } -} - func GetHash(s string) string { h := sha1.New() h.Write([]byte(s)) diff --git a/main.go b/main.go index 79cb9fb..5777ea2 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,8 @@ import ( "bytes" "context" "encoding/json" + "flag" + "fmt" "io/ioutil" "net/http" "os" @@ -17,6 +19,26 @@ import ( "github.com/fatih/color" ) +var ( + threadsFlag = flag.Int("threads", core.DefaultThreads, "Number of concurrent threads (default number of logical CPUs)") + silentFlag = flag.Bool("silent", core.DefaultSilent, "Suppress all output except for errors") + debugFlag = flag.Bool("debug", core.DefaultDebug, "Print debugging information") + maxRepoFlag = flag.Uint("maximum-repository-size", core.DefaultMaximumRepositorySize, "Maximum repository size to process in KB") + maxFileSizeFlag = flag.Uint("maximum-file-size", core.DefaultMaximumFileSize, "Maximum file size to process in KB") + cloneTimeoutFlag = flag.Uint("clone-repository-timeout", core.DefaultCloneRepositoryTimeout, "Maximum time it should take to clone a repository in seconds. Increase this if you have a slower connection") + entropyFlag = flag.Float64("entropy-threshold", core.DefaultEntropy, "Set to 0 to disable entropy checks") + minStarsFlag = flag.Uint("minimum-stars", core.DefaultMinimumStars, "Only process repositories with this many stars. Default 0 will ignore star count") + pathChecksFlag = flag.Bool("path-checks", core.DefaultPathChecks, "Set to false to disable checking of filepaths, i.e. just match regex patterns of file contents") + gistsFlag = flag.Bool("process-gists", core.DefaultProcessGists, "Will watch and process Gists. Set to false to disable.") + tempDirFlag = flag.String("temp-directory", core.DefaultTempDirectory, "Directory to process and store repositories/matches") + csvFlag = flag.String("csv-path", "", "CSV file path to log found secrets to. Leave blank to disable") + searchQueryFlag = flag.String("search-query", "", "Specify a search string to ignore signatures and filter on files containing this string (regex compatible)") + localFlag = flag.String("local", "", "Specify local directory (absolute path) which to scan. Scans only given directory recursively. No need to have GitHub tokens with local run.") + liveFlag = flag.String("live", "", "Your shhgit live endpoint") + configPathFlag = flag.String("config-path", "", "Searches for config.yaml from given directory. If not set, tries to find if from shhgit binary's and current directory") + configNameFlag = flag.String("config-name", "config.yaml", "filename to search for") +) + type MatchEvent struct { Url string Matches []string @@ -26,62 +48,62 @@ type MatchEvent struct { Source core.GitResourceType } -var session = core.GetSession() +var session *core.Session -func ProcessRepositories() { - threadNum := *session.Options.Threads +func ProcessRepositories(s *core.Session) { + threadNum := *s.Options.Threads for i := 0; i < threadNum; i++ { go func(tid int) { for { - timeout := time.Duration(*session.Options.CloneRepositoryTimeout) * time.Second + timeout := time.Duration(*s.Options.CloneRepositoryTimeout) * time.Second _, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - repository := <-session.Repositories + repository := <-s.Repositories repo, err := core.GetRepository(session, repository.Id) if err != nil { - session.Log.Warn("Failed to retrieve repository %d: %s", repository.Id, err) + s.Log.Warn("Failed to retrieve repository %d: %s", repository.Id, err) continue } if repo.GetPermissions()["pull"] && - uint(repo.GetStargazersCount()) >= *session.Options.MinimumStars && - uint(repo.GetSize()) < *session.Options.MaximumRepositorySize { + uint(repo.GetStargazersCount()) >= *s.Options.MinimumStars && + uint(repo.GetSize()) < *s.Options.MaximumRepositorySize { - processRepositoryOrGist(repo.GetCloneURL(), repository.Ref, repo.GetStargazersCount(), core.GITHUB_SOURCE) + processRepositoryOrGist(s, repo.GetCloneURL(), repository.Ref, repo.GetStargazersCount(), core.GITHUB_SOURCE) } } }(i) } } -func ProcessGists() { - threadNum := *session.Options.Threads +func ProcessGists(s *core.Session) { + threadNum := *s.Options.Threads for i := 0; i < threadNum; i++ { go func(tid int) { for { - gistUrl := <-session.Gists - processRepositoryOrGist(gistUrl, "", -1, core.GIST_SOURCE) + gistUrl := <-s.Gists + processRepositoryOrGist(s, gistUrl, "", -1, core.GIST_SOURCE) } }(i) } } -func ProcessComments() { - threadNum := *session.Options.Threads +func ProcessComments(s *core.Session) { + threadNum := *s.Options.Threads for i := 0; i < threadNum; i++ { go func(tid int) { for { - commentBody := <-session.Comments - dir := core.GetTempDir(core.GetHash(commentBody)) + commentBody := <-s.Comments + dir := core.GetTempDir(s, core.GetHash(commentBody)) ioutil.WriteFile(filepath.Join(dir, "comment.ignore"), []byte(commentBody), 0644) - if !checkSignatures(dir, "ISSUE", 0, core.GITHUB_COMMENT) { + if !checkSignatures(s, dir, "ISSUE", 0, core.GITHUB_COMMENT) { os.RemoveAll(dir) } } @@ -89,41 +111,42 @@ func ProcessComments() { } } -func processRepositoryOrGist(url string, ref string, stars int, source core.GitResourceType) { +func processRepositoryOrGist(s *core.Session, url string, ref string, stars int, source core.GitResourceType) { var ( matchedAny bool = false ) - dir := core.GetTempDir(core.GetHash(url)) + dir := core.GetTempDir(s, core.GetHash(url)) _, err := core.CloneRepository(session, url, ref, dir) if err != nil { - session.Log.Debug("[%s] Cloning failed: %s", url, err.Error()) + s.Log.Debug("[%s] Cloning failed: %s", url, err.Error()) os.RemoveAll(dir) return } - session.Log.Debug("[%s] Cloning %s in to %s", url, ref, strings.Replace(dir, *session.Options.TempDirectory, "", -1)) - matchedAny = checkSignatures(dir, url, stars, source) + s.Log.Debug("[%s] Cloning %s in to %s", url, ref, strings.Replace(dir, *s.Options.TempDirectory, "", -1)) + matchedAny = checkSignatures(s, dir, url, stars, source) if !matchedAny { os.RemoveAll(dir) } } -func checkSignatures(dir string, url string, stars int, source core.GitResourceType) (matchedAny bool) { - for _, file := range core.GetMatchingFiles(dir) { +func checkSignatures(s *core.Session, dir string, url string, stars int, source core.GitResourceType) (matchedAny bool) { + + for _, file := range core.GetMatchingFiles(s, dir) { var ( matches []string relativeFileName string ) - if strings.Contains(dir, *session.Options.TempDirectory) { - relativeFileName = strings.Replace(file.Path, *session.Options.TempDirectory, "", -1) + if strings.Contains(dir, *s.Options.TempDirectory) { + relativeFileName = strings.Replace(file.Path, *s.Options.TempDirectory, "", -1) } else { relativeFileName = strings.Replace(file.Path, dir, "", -1) } - if *session.Options.SearchQuery != "" { - queryRegex := regexp.MustCompile(*session.Options.SearchQuery) + if *s.Options.SearchQuery != "" { + queryRegex := regexp.MustCompile(*s.Options.SearchQuery) for _, match := range queryRegex.FindAllSubmatch(file.Contents, -1) { matches = append(matches, string(match[0])) } @@ -131,32 +154,32 @@ func checkSignatures(dir string, url string, stars int, source core.GitResourceT if matches != nil { count := len(matches) m := strings.Join(matches, ", ") - session.Log.Important("[%s] %d %s for %s in file %s: %s", url, count, core.Pluralize(count, "match", "matches"), color.GreenString("Search Query"), relativeFileName, color.YellowString(m)) - session.WriteToCsv([]string{url, "Search Query", relativeFileName, m}) + s.Log.Important("[%s] %d %s for %s in file %s: %s", url, count, core.Pluralize(count, "match", "matches"), color.GreenString("Search Query"), relativeFileName, color.YellowString(m)) + s.WriteToCsv([]string{url, "Search Query", relativeFileName, m}) } } else { - for _, signature := range session.Signatures { + for _, signature := range s.Signatures { if matched, part := signature.Match(file); matched { if part == core.PartContents { - if matches = signature.GetContentsMatches(file.Contents); len(matches) > 0 { + if matches = signature.GetContentsMatches(s, file.Contents); len(matches) > 0 { count := len(matches) m := strings.Join(matches, ", ") - publish(&MatchEvent{Source: source, Url: url, Matches: matches, Signature: signature.Name(), File: relativeFileName, Stars: stars}) + publish(s, &MatchEvent{Source: source, Url: url, Matches: matches, Signature: signature.Name(), File: relativeFileName, Stars: stars}) matchedAny = true - session.Log.Important("[%s] %d %s for %s in file %s: %s", url, count, core.Pluralize(count, "match", "matches"), color.GreenString(signature.Name()), relativeFileName, color.YellowString(m)) - session.WriteToCsv([]string{url, signature.Name(), relativeFileName, m}) + s.Log.Important("[%s] %d %s for %s in file %s: %s", url, count, core.Pluralize(count, "match", "matches"), color.GreenString(signature.Name()), relativeFileName, color.YellowString(m)) + s.WriteToCsv([]string{url, signature.Name(), relativeFileName, m}) } } else { - if *session.Options.PathChecks { - publish(&MatchEvent{Source: source, Url: url, Matches: matches, Signature: signature.Name(), File: relativeFileName, Stars: stars}) + if *s.Options.PathChecks { + publish(s, &MatchEvent{Source: source, Url: url, Matches: matches, Signature: signature.Name(), File: relativeFileName, Stars: stars}) matchedAny = true - session.Log.Important("[%s] Matching file %s for %s", url, color.YellowString(relativeFileName), color.GreenString(signature.Name())) - session.WriteToCsv([]string{url, signature.Name(), relativeFileName, ""}) + s.Log.Important("[%s] Matching file %s for %s", url, color.YellowString(relativeFileName), color.GreenString(signature.Name())) + s.WriteToCsv([]string{url, signature.Name(), relativeFileName, ""}) } - if *session.Options.EntropyThreshold > 0 && file.CanCheckEntropy() { + if *s.Options.EntropyThreshold > 0 && file.CanCheckEntropy(s) { scanner := bufio.NewScanner(bytes.NewReader(file.Contents)) for scanner.Scan() { @@ -165,21 +188,21 @@ func checkSignatures(dir string, url string, stars int, source core.GitResourceT if len(line) > 6 && len(line) < 100 { entropy := core.GetEntropy(line) - if entropy >= *session.Options.EntropyThreshold { + if entropy >= *s.Options.EntropyThreshold { blacklistedMatch := false - for _, blacklistedString := range session.Config.BlacklistedStrings { + for _, blacklistedString := range s.Config.BlacklistedStrings { if strings.Contains(strings.ToLower(line), strings.ToLower(blacklistedString)) { blacklistedMatch = true } } if !blacklistedMatch { - publish(&MatchEvent{Source: source, Url: url, Matches: []string{line}, Signature: "High entropy string", File: relativeFileName, Stars: stars}) + publish(s, &MatchEvent{Source: source, Url: url, Matches: []string{line}, Signature: "High entropy string", File: relativeFileName, Stars: stars}) matchedAny = true - session.Log.Important("[%s] Potential secret in %s = %s", url, color.YellowString(relativeFileName), color.GreenString(line)) - session.WriteToCsv([]string{url, "High entropy string", relativeFileName, line}) + s.Log.Important("[%s] Potential secret in %s = %s", url, color.YellowString(relativeFileName), color.GreenString(line)) + s.WriteToCsv([]string{url, "High entropy string", relativeFileName, line}) } } } @@ -190,51 +213,78 @@ func checkSignatures(dir string, url string, stars int, source core.GitResourceT } } - if !matchedAny && len(*session.Options.Local) <= 0 { + if !matchedAny && len(*s.Options.Local) <= 0 { os.Remove(file.Path) } } return } -func publish(event *MatchEvent) { +func publish(s *core.Session, event *MatchEvent) { // todo: implement a modular plugin system to handle the various outputs (console, live, csv, webhooks, etc) - if len(*session.Options.Live) > 0 { + if len(*s.Options.Live) > 0 { data, _ := json.Marshal(event) - http.Post(*session.Options.Live, "application/json", bytes.NewBuffer(data)) + http.Post(*s.Options.Live, "application/json", bytes.NewBuffer(data)) } } func main() { - session.Log.Info(color.HiBlueString(core.Banner)) - session.Log.Info("\t%s\n", color.HiCyanString(core.Author)) - session.Log.Info("[*] Loaded %s signatures. Using %s worker threads. Temp work dir: %s\n", color.BlueString("%d", len(session.Signatures)), color.BlueString("%d", *session.Options.Threads), color.BlueString(*session.Options.TempDirectory)) + flag.Parse() + ctx := context.Background() + + s, err := core.NewSession(ctx, &core.Options{ + Threads: threadsFlag, + Silent: silentFlag, + Debug: debugFlag, + MaximumRepositorySize: maxRepoFlag, + MaximumFileSize: maxFileSizeFlag, + CloneRepositoryTimeout: cloneTimeoutFlag, + EntropyThreshold: entropyFlag, + MinimumStars: minStarsFlag, + PathChecks: pathChecksFlag, + ProcessGists: gistsFlag, + TempDirectory: tempDirFlag, + CSVPath: csvFlag, + SearchQuery: searchQueryFlag, + Local: localFlag, + Live: liveFlag, + ConfigPath: configPathFlag, + ConfigName: configNameFlag, + }) + + if err != nil { + fmt.Printf("Failed to create session: %v\n", err) + os.Exit(1) + } + + s.Log.Info(color.HiBlueString(core.Banner)) + s.Log.Info("\t%s\n", color.HiCyanString(core.Author)) + s.Log.Info("[*] Loaded %s signatures. Using %s worker threads. Temp work dir: %s\n", color.BlueString("%d", len(s.Signatures)), color.BlueString("%d", *s.Options.Threads), color.BlueString(*s.Options.TempDirectory)) - if len(*session.Options.Local) > 0 { - session.Log.Info("[*] Scanning local directory: %s - skipping public repository checks...", color.BlueString(*session.Options.Local)) + if s.Options.Local != nil { + s.Log.Info("[*] Scanning local directory: %s - skipping public repository checks...", color.BlueString(*s.Options.Local)) rc := 0 - if checkSignatures(*session.Options.Local, *session.Options.Local, -1, core.LOCAL_SOURCE) { + if checkSignatures(s, *s.Options.Local, *s.Options.Local, -1, core.LOCAL_SOURCE) { rc = 1 } else { - session.Log.Info("[*] No matching secrets found in %s!", color.BlueString(*session.Options.Local)) + s.Log.Info("[*] No matching secrets found in %s!", color.BlueString(*s.Options.Local)) } os.Exit(rc) - } else { - if *session.Options.SearchQuery != "" { - session.Log.Important("Search Query '%s' given. Only returning matching results.", *session.Options.SearchQuery) - } - - go core.GetRepositories(session) - go ProcessRepositories() - go ProcessComments() + } + if *s.Options.SearchQuery != "" { + s.Log.Important("Search Query '%s' given. Only returning matching results.", *s.Options.SearchQuery) + } - if *session.Options.ProcessGists { - go core.GetGists(session) - go ProcessGists() - } + go core.GetRepositories(s) + go ProcessRepositories(s) + go ProcessComments(s) - spinny := core.ShowSpinner() - select {} - spinny() + if *s.Options.ProcessGists { + go core.GetGists(s) + go ProcessGists(s) } + + spinny := core.ShowSpinner() + select {} + spinny() }