feat: restructured pillager for modularity and expansion

.github/workflows/gobuild.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.21
 
       - name: Build executables
         run: |

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.21
       - name: Run GoReleaser and release executables
         uses: goreleaser/goreleaser-action@v2
         with:

.github/workflows/test.yml

@@ -12,7 +12,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.21
 
       - name: Test
         run: |

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17-alpine
+FROM golang:1.21-alpine
 
 WORKDIR /src/
 COPY . /src/

README.md

@@ -22,7 +22,7 @@ Pillager is designed to provide a simple means of leveraging Go's strong concurr
 directories for sensitive information in files. Pillager does this by standing on the shoulders
 of [a few giants](#shoulders-of-giants). Once pillager finds files that match the specified pattern, the file is scanned
 using a series of concurrent workers that each take a line of the file from the job queue and hunt for sensitive pattern
-matches. The available pattern filters can be defined in a rules.toml file or you can use the default ruleset.
+matches. The available pattern filters can be defined in a pillager.toml file or you can use the default ruleset.
 
 ## Installation
 
@@ -31,7 +31,7 @@ matches. The available pattern filters can be defined in a rules.toml file or yo
 If you have Go setup on your system, you can install Pillager with `go get`
 
 ```shell script
-go get github.com/brittonhayes/pillager
+go install github.com/brittonhayes/pillager@latest
 ```
 
 ### Scoop (Windows)
@@ -79,29 +79,56 @@ Pillager provides a terminal user interface built with [bubbletea](https://githu
 ### Gitleaks Rules
 
 Pillager provides full support for Gitleaks[^2] rules. This can either be passed
-in with a rules.toml[^1] file, or you can use the default ruleset by leaving the rules flag blank.
+in with a rules[^1] section in your pillager.toml file, or you can use the default ruleset by leaving the config flag blank.
 
 [^1]: [Gitleaks Rules Reference](https://github.com/zricethezav/gitleaks/blob/57f9bc83d169bea363f2990a4de334b54efc3d7d/config/gitleaks.toml)
 
 ```toml
-# rules.toml
-title = "pillager rules"
+# pillager.toml
+# Basic configuration
+verbose = false 
+path = "."
+workers = 4
+redact = true
+reporter = "json-pretty"
+
+# Rules for secret detection
+[[rules]]
+description = "AWS Access Key"
+id = "aws-access-key"
+regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
+tags = ["aws", "credentials"]
 
 [[rules]]
-id = "gitlab-pat"
-description = "GitLab Personal Access Token"
-regex = '''glpat-[0-9a-zA-Z\-\_]{20}'''
+description = "AWS Secret Key"
+id = "aws-secret-key"
+regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]'''
+tags = ["aws", "credentials"]
 
 [[rules]]
-id = "aws-access-token"
-description = "AWS"
-regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}'''
+description = "GitHub Token"
+id = "github-token"
+regex = '''ghp_[0-9a-zA-Z]{36}'''
+tags = ["github", "token"]
 
-# Cryptographic keys
 [[rules]]
-id = "PKCS8-PK"
-description = "PKCS8 private key"
-regex = '''-----BEGIN PRIVATE KEY-----'''
+description = "Private Key"
+id = "private-key"
+regex = '''-----BEGIN (?:RSA|OPENSSH|DSA|EC|PGP) PRIVATE KEY( BLOCK)?-----'''
+tags = ["key", "private"]
+
+# Allowlist configuration
+[allowlist]
+paths = [
+    ".*/_test\\.go$",
+    ".*/testdata/.*",
+    ".*\\.md$",
+    ".*/vendor/.*"
+]
+regexes = [
+    "EXAMPLE_KEY",
+    "DUMMY_SECRET"
+] 
 ```
 
 ### Built-in Output Formats
@@ -132,10 +159,10 @@ pillager hunt ./example -f json | jq
 pillager hunt . -f yaml
 ```
 
-#### TOML
+#### JSON Pretty
 
 ```shell
-pillager hunt . -f toml
+pillager hunt . -f json-pretty
 ```
 
 #### HTML
@@ -171,14 +198,14 @@ pillager hunt . --template "{{ range .}}Secret: {{.Secret}}{{end}}"
 #### Custom Go Template from File
 
 ```shell
-pillager hunt . -t "$(cat pkg/templates/simple.tmpl)"
+pillager hunt . -t "$(cat internal/templates/simple.tmpl)"
 ```
 
 </details>
 
 ### Custom Templates
 
-Pillager allows you to use powerful `go text/template` to customize the output format. Here are a few template examples.
+Pillager allows you to use powerful `go text/template` and [sprig](https://masterminds.github.io/sprig/) functions to customize the output format. Here are a few template examples.
 
 #### Basic
 
@@ -203,16 +230,11 @@ Pillager allows you to use powerful `go text/template` to customize the output f
 
 ```
 
-> More template examples can be found in the [templates](./pkg/templates) directory.
+> More template examples can be found in the [templates](./internal/templates) directory.
 
 ## Documentation
 
-:books: [View the docs](pkg/hunter)
-
-GoDoc documentation is available on [pkg.go.dev for pillager](https://pkg.go.dev/github.com/brittonhayes/pillager) but
-it is also available for all packages in the repository in markdown format. Just open the folder of any package, and
-you'll see the GoDocs rendered in beautiful Github-flavored markdown thanks to the
-awesome [gomarkdoc](https://github.com/princjef/gomarkdoc) tool.
+GoDoc documentation is available on [pkg.go.dev for pillager](https://pkg.go.dev/github.com/brittonhayes/pillager). 
 
 ## Development
 
@@ -237,7 +259,7 @@ If you've seen a CLI written in Go before, there's a pretty high chance it was b
 library enough. It empowers developers to make consistent, dynamic, and self-documenting command line tools with ease.
 Some examples include `kubectl`, `hugo`, and Github's `gh` CLI.
 
-#### [Gitleaks](https://github.com/zricethezav/gitleaks)
+#### [Gitleaks](https://github.com/gitleaks/gitleaks)
 
 **What is Gitleaks?**
 
@@ -248,9 +270,9 @@ it's worth your time to check it out.
 
 **Why is Gitleaks relevant to Pillager?**
 
-[^2]: [Gitleaks](https://github.com/zricethezav/gitleaks)
+[^2]: [Gitleaks](https://github.com/gitleaks/gitleaks)
 
-Pillager implements the powerful [rules](https://github.com/zricethezav/gitleaks#rules-summary) functionality of
+Pillager implements the powerful [rules](https://github.com/gitleaks/gitleaks#rules-summary) functionality of
 Gitleaks while taking a different approach to presenting and handling the secrets found. While I have provided a
 baseline set of default rules, Pillager becomes much more powerful if you allow users to create rules for their own
 use-cases.

_examples/scanner/main.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/brittonhayes/pillager"
+	"github.com/brittonhayes/pillager/pkg/scanner"
+)
+
+func main() {
+	// Set scanner options
+	opts := pillager.Options{
+		Path:     ".",
+		Workers:  runtime.NumCPU(),
+		Reporter: "json",
+		Redact:   true,
+	}
+
+	// Create a new gitleaks scanner
+	s, _ := scanner.NewGitleaksScanner(opts)
+
+	// Scan the current directory
+	results, _ := s.Scan(opts.Path)
+
+	// Report results
+	fmt.Println(results)
+}

go.mod

@@ -1,6 +1,6 @@
 module github.com/brittonhayes/pillager
 
-go 1.18
+go 1.21
 
 require (
 	github.com/BurntSushi/toml v1.1.0

internal/commands/hunt.go

@@ -9,16 +9,18 @@ import (
 	"runtime"
 
 	"github.com/brittonhayes/pillager"
-	"github.com/brittonhayes/pillager/internal/scanner"
+	"github.com/brittonhayes/pillager/pkg/scanner"
+	"github.com/brittonhayes/pillager/pkg/tui/model"
+	tea "github.com/charmbracelet/bubbletea"
 	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 )
 
 var (
 	verbose     bool
 	redact      bool
 	level       string
-	rulesConfig string
 	reporter    string
 	templ       string
 	workers     int
@@ -56,7 +58,6 @@ var huntCmd = &cobra.Command{
 	Custom Go Template Format from Template File:
 		pillager hunt ./example --template "$(cat pkg/templates/simple.tmpl)"
 `,
-	Args: cobra.ExactArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		if level != "" {
 			lvl, err := zerolog.ParseLevel(level)
@@ -73,7 +74,6 @@ var huntCmd = &cobra.Command{
 				return fmt.Errorf("failed to load config: %w", err)
 			}
 			opts = &pillager.Options{
-				ScanPath: args[0],
 				Workers:  runtime.NumCPU(),
 				Verbose:  false,
 				Template: "",
@@ -82,9 +82,15 @@ var huntCmd = &cobra.Command{
 			}
 		}
 
+		// Get path from args if provided, otherwise use config path
+		scanPath := ""
+		if len(args) > 0 {
+			scanPath = args[0]
+		}
+
 		// Merge command line flags with config file
 		flagOpts := &pillager.Options{
-			ScanPath: args[0],
+			Path:     scanPath,
 			Redact:   redact,
 			Verbose:  verbose,
 			Workers:  workers,
@@ -93,29 +99,40 @@ var huntCmd = &cobra.Command{
 		}
 		configLoader.MergeWithFlags(opts, flagOpts)
 
+		// Check if path is provided either via args or config
+		if opts.Path == "" {
+			return fmt.Errorf("scan path must be provided either as an argument or in the config file")
+		}
+
 		s, err := scanner.NewGitleaksScanner(*opts)
 		if err != nil {
 			return fmt.Errorf("failed to create scanner: %w", err)
 		}
 
-		// if interactive {
-		// 	return runInteractive(scan)
-		// }
+		if interactive {
+			return runInteractive(s)
+		}
 
-		results, err := s.Scan(args[0])
+		results, err := s.Scan()
 		if err != nil {
 			return err
 		}
 
+		if len(results) == 0 {
+			fmt.Println("[]")
+			log.Debug().Msg("no secrets or sensitive information were found at the target directory")
+			return nil
+		}
+
 		return s.Reporter().Report(os.Stdout, results)
 	},
 }
 
-// func runInteractive(h *scanner.Hunter) error {
-// 	m := model.NewModel(h)
-// 	p := tea.NewProgram(m, tea.WithAltScreen())
-// 	return p.Start()
-// }
+func runInteractive(h scanner.Scanner) error {
+	m := model.NewModel(h)
+	p := tea.NewProgram(m, tea.WithAltScreen())
+	return p.Start()
+}
 
 func init() {
 	rootCmd.AddCommand(huntCmd)
@@ -124,8 +141,7 @@ func init() {
 	huntCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "enable scanner verbose output")
 	huntCmd.Flags().StringVarP(&level, "log-level", "l", "info", "set logging level")
 	huntCmd.Flags().StringVarP(&config, "config", "c", "", "path to pillager config file")
-	huntCmd.Flags().StringVarP(&rulesConfig, "rules", "r", "", "path to gitleaks rules.toml config")
-	huntCmd.Flags().StringVarP(&reporter, "format", "f", "json", "set secret reporter format (json, yaml)")
+	huntCmd.Flags().StringVarP(&reporter, "format", "f", "json-pretty", "set secret reporter format (json, yaml, html, html-table, table, markdown)")
 	huntCmd.Flags().BoolVar(&redact, "redact", false, "redact secret from results")
 	huntCmd.Flags().StringVarP(&templ, "template", "t", "", "set go text/template string for output format")
 }