Add support for centralized allowlists

cmd/crowdsec-cli/clidecision/decisions.go

@@ -317,7 +317,8 @@
 	return cmd
 }
 
-func (cli *cliDecisions) add(ctx context.Context, addIP, addRange, addDuration, addValue, addScope, addReason, addType string) error {
+//nolint:revive // we'll reduce the number of args later
+func (cli *cliDecisions) add(ctx context.Context, addIP, addRange, addDuration, addValue, addScope, addReason, addType string, bypassAllowlist bool) error {
 	alerts := models.AddAlertsRequest{}
 	origin := types.CscliOrigin
 	capacity := int32(0)
@@ -350,6 +351,15 @@
 		addReason = fmt.Sprintf("manual '%s' from '%s'", addType, cli.cfg().API.Client.Credentials.Login)
 	}
 
+	if !bypassAllowlist && (addScope == types.Ip || addScope == types.Range) {
+		resp, _, err := cli.client.Allowlists.CheckIfAllowlistedWithReason(ctx, addValue)
+		if err != nil {
+			log.Errorf("Cannot check if %s is in allowlist: %s", addValue, err)
+		} else if resp.Allowlisted {
+			return fmt.Errorf("%s is allowlisted by item %s, use --bypass-allowlist to add the decision anyway", addValue, resp.Reason)
+		}
+	}
+
 	decision := models.Decision{
 		Duration: &addDuration,
 		Scope:    &addScope,
@@ -398,13 +408,14 @@
 
 func (cli *cliDecisions) newAddCmd() *cobra.Command {
 	var (
-		addIP       string
-		addRange    string
-		addDuration string
-		addValue    string
-		addScope    string
-		addReason   string
-		addType     string
+		addIP           string
+		addRange        string
+		addDuration     string
+		addValue        string
+		addScope        string
+		addReason       string
+		addType         string
+		bypassAllowlist bool
 	)
 
 	cmd := &cobra.Command{
@@ -419,7 +430,7 @@
 		Args:              cobra.NoArgs,
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			return cli.add(cmd.Context(), addIP, addRange, addDuration, addValue, addScope, addReason, addType)
+			return cli.add(cmd.Context(), addIP, addRange, addDuration, addValue, addScope, addReason, addType, bypassAllowlist)
 		},
 	}
 
@@ -432,6 +443,7 @@
 	flags.StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
 	flags.StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
 	flags.StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
+	flags.BoolVarP(&bypassAllowlist, "bypass-allowlist", "B", false, "Add decision even if value is in allowlist")
 
 	return cmd
 }

cmd/crowdsec-cli/main.go

@@ -12,9 +12,11 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/go-cs-lib/ptr"
 	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clialert"
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/cliallowlists"
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clibouncer"
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clicapi"
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/cliconfig"
@@ -166,6 +168,8 @@ func (cli *cliRoot) initialize() error {
 		}
 	}
 
+	csConfig.DbConfig.LogLevel = ptr.Of(cli.wantedLogLevel())
+
 	return nil
 }
 
@@ -282,6 +286,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	cmd.AddCommand(cliitem.NewContext(cli.cfg).NewCommand())
 	cmd.AddCommand(cliitem.NewAppsecConfig(cli.cfg).NewCommand())
 	cmd.AddCommand(cliitem.NewAppsecRule(cli.cfg).NewCommand())
+	cmd.AddCommand(cliallowlists.New(cli.cfg).NewCommand())
 
 	cli.addSetup(cmd)
 

cmd/crowdsec/crowdsec.go

@@ -14,6 +14,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
+	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
@@ -23,7 +24,7 @@ import (
 )
 
 // initCrowdsec prepares the log processor service
-func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, []acquisition.DataSource, error) {
+func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub, testMode bool) (*parser.Parsers, []acquisition.DataSource, error) {
 	var err error
 
 	if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil {
@@ -51,6 +52,17 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, []
 		return nil, nil, err
 	}
 
+	if !testMode {
+		err = apiclient.InitLAPIClient(
+			context.TODO(), cConfig.API.Client.Credentials.URL, cConfig.API.Client.Credentials.PapiURL,
+			cConfig.API.Client.Credentials.Login, cConfig.API.Client.Credentials.Password,
+			hub.GetInstalledListForAPI())
+
+		if err != nil {
+			return nil, nil, fmt.Errorf("while initializing LAPIClient: %w", err)
+		}
+	}
+
 	datasources, err := LoadAcquisition(cConfig)
 	if err != nil {
 		return nil, nil, fmt.Errorf("while loading acquisition config: %w", err)
@@ -116,7 +128,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H
 	})
 	bucketWg.Wait()
 
-	apiClient, err := AuthenticatedLAPIClient(context.TODO(), *cConfig.API.Client.Credentials, hub)
+	apiClient, err := apiclient.GetLAPIClient()
 	if err != nil {
 		return err
 	}

cmd/crowdsec/serve.go

@@ -94,7 +94,7 @@
 			return nil, err
 		}
 
-		csParsers, datasources, err := initCrowdsec(cConfig, hub)
+		csParsers, datasources, err := initCrowdsec(cConfig, hub, false)
 		if err != nil {
 			return nil, fmt.Errorf("unable to init crowdsec: %w", err)
 		}
@@ -396,7 +396,7 @@
 			return err
 		}
 
-		csParsers, datasources, err := initCrowdsec(cConfig, hub)
+		csParsers, datasources, err := initCrowdsec(cConfig, hub, flags.TestMode)
 		if err != nil {
 			return fmt.Errorf("crowdsec init: %w", err)
 		}

pkg/acquisition/modules/appsec/appsec.go

@@ -20,6 +20,7 @@
 	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
+	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/appsec"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -31,6 +32,8 @@
 )
 
 var DefaultAuthCacheDuration = (1 * time.Minute)
+var negativeAllowlistCacheDuration = (5 * time.Minute)
+var positiveAllowlistCacheDuration = (5 * time.Minute)
 
 // configuration structure of the acquis for the application security engine
 type AppsecSourceConfig struct {
@@ -49,18 +52,20 @@
 
 // runtime structure of AppsecSourceConfig
 type AppsecSource struct {
-	metricsLevel  int
-	config        AppsecSourceConfig
-	logger        *log.Entry
-	mux           *http.ServeMux
-	server        *http.Server
-	outChan       chan types.Event
-	InChan        chan appsec.ParsedRequest
-	AppsecRuntime *appsec.AppsecRuntimeConfig
-	AppsecConfigs map[string]appsec.AppsecConfig
-	lapiURL       string
-	AuthCache     AuthCache
-	AppsecRunners []AppsecRunner // one for each go-routine
+	metricsLevel   int
+	config         AppsecSourceConfig
+	logger         *log.Entry
+	mux            *http.ServeMux
+	server         *http.Server
+	outChan        chan types.Event
+	InChan         chan appsec.ParsedRequest
+	AppsecRuntime  *appsec.AppsecRuntimeConfig
+	AppsecConfigs  map[string]appsec.AppsecConfig
+	lapiURL        string
+	AuthCache      AuthCache
+	AppsecRunners  []AppsecRunner // one for each go-routine
+	allowlistCache allowlistCache
+	apiClient      *apiclient.ApiClient
 }
 
 // Struct to handle cache of authentication
@@ -69,6 +74,17 @@
 	mu      sync.RWMutex
 }
 
+// FIXME: auth and allowlist should probably be merged to a common structure
+type allowlistCache struct {
+	mu        sync.RWMutex
+	allowlist map[string]allowlistCacheEntry
+}
+
+type allowlistCacheEntry struct {
+	allowlisted bool
+	expiration  time.Time
+}
+
 func NewAuthCache() AuthCache {
 	return AuthCache{
 		APIKeys: make(map[string]time.Time, 0),
@@ -90,6 +106,30 @@
 	return expiration, exists
 }
 
+func NewAllowlistCache() allowlistCache {
+	return allowlistCache{
+		allowlist: make(map[string]allowlistCacheEntry, 0),
+		mu:        sync.RWMutex{},
+	}
+}
+
+func (ac *allowlistCache) Set(value string, allowlisted bool, expiration time.Time) {
+	ac.mu.Lock()
+	ac.allowlist[value] = allowlistCacheEntry{
+		allowlisted: allowlisted,
+		expiration:  expiration,
+	}
+	ac.mu.Unlock()
+}
+
+func (ac *allowlistCache) Get(value string) (bool, time.Time, bool) {
+	ac.mu.RLock()
+	entry, exists := ac.allowlist[value]
+	ac.mu.RUnlock()
+
+	return entry.allowlisted, entry.expiration, exists
+}
+
 // @tko + @sbl : we might want to get rid of that or improve it
 type BodyResponse struct {
 	Action string `json:"action"`
@@ -246,6 +286,12 @@
 	// We don´t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec
 	w.mux.HandleFunc(w.config.Path, w.appsecHandler)
 
+	w.apiClient, err = apiclient.GetLAPIClient()
+	if err != nil {
+		return fmt.Errorf("unable to get authenticated LAPI client: %w", err)
+	}
+	w.allowlistCache = NewAllowlistCache()
+
 	return nil
 }
 
@@ -377,6 +423,33 @@
 	return resp.StatusCode == http.StatusOK
 }
 
+func (w *AppsecSource) isAllowlisted(ctx context.Context, value string, query bool) bool {
+	var err error
+
+	allowlisted, expiration, exists := w.allowlistCache.Get(value)
+	if exists && !time.Now().After(expiration) {
+		return allowlisted
+	}
+
+	if !query {
+		return false
+	}
+
+	allowlisted, _, err = w.apiClient.Allowlists.CheckIfAllowlisted(ctx, value)
+	if err != nil {
+		w.logger.Errorf("unable to check if %s is allowlisted: %s", value, err)
+		return false
+	}
+
+	if allowlisted {
+		w.allowlistCache.Set(value, allowlisted, time.Now().Add(positiveAllowlistCacheDuration))
+	} else {
+		w.allowlistCache.Set(value, allowlisted, time.Now().Add(negativeAllowlistCacheDuration))
+	}
+
+	return allowlisted
+}
+
 // should this be in the runner ?
 func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) {
 	w.logger.Debugf("Received request from '%s' on %s", r.RemoteAddr, r.URL.Path)
@@ -406,6 +479,25 @@
 		w.AuthCache.Set(apiKey, time.Now().Add(*w.config.AuthCacheDuration))
 	}
 
+	// check if the client IP is allowlisted
+	if w.isAllowlisted(r.Context(), clientIP, false) {
+		w.logger.Infof("%s is allowlisted by LAPI, not processing", clientIP)
+		statusCode, appsecResponse := w.AppsecRuntime.GenerateResponse(appsec.AppsecTempResponse{
+			InBandInterrupt:    false,
+			OutOfBandInterrupt: false,
+			Action:             appsec.AllowRemediation,
+		}, w.logger)
+		body, err := json.Marshal(appsecResponse)
+		if err != nil {
+			w.logger.Errorf("unable to serialize response: %s", err)
+			rw.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		rw.WriteHeader(statusCode)
+		rw.Write(body)
+		return
+	}
+
 	// parse the request only once
 	parsedRequest, err := appsec.NewParsedRequestFromRequest(r, w.logger)
 	if err != nil {

pkg/acquisition/modules/loki/internal/lokiclient/loki_client.go

@@ -205,6 +205,7 @@ func (lc *LokiClient) getURLFor(endpoint string, params map[string]string) strin
 func (lc *LokiClient) Ready(ctx context.Context) error {
 	tick := time.NewTicker(500 * time.Millisecond)
 	url := lc.getURLFor("ready", nil)
+	lc.Logger.Debugf("Using url: %s for ready check", url)
 	for {
 		select {
 		case <-ctx.Done():