Skip to content

Commit

Permalink
feat: Add nginx-inc ingress controller support
Browse files Browse the repository at this point in the history
Signed-off-by: Anurag Rajawat <[email protected]>
  • Loading branch information
anurag-rajawat committed Oct 21, 2024
1 parent 7cb1052 commit b729367
Show file tree
Hide file tree
Showing 7 changed files with 288 additions and 56 deletions.
12 changes: 12 additions & 0 deletions deployments/sentryflow.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,18 @@ rules:
- delete
resources:
- wasmplugins
- apiGroups:
- ""
verbs:
- get
resources:
- configmaps
- apiGroups:
- apps
verbs:
- get
resources:
- deployments
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
Expand Down
34 changes: 20 additions & 14 deletions sentryflow/config/default.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,28 @@ filters:
server:
port: 8081

envoy:
uri: anuragrajawat/httpfilter:v0.1
# Envoy filter is required for `istio-sidecar` service-mesh receiver.
# envoy:
# uri: 5gsec/sentryflow-httpfilter:latest

# Following is required for `nginx-inc-ingress-controller` receiver.
# nginxIngress:
# deploymentName: nginx-ingress-controller
# configMapName: nginx-ingress
# sentryFlowNjsConfigMapName: sentryflow-njs

receivers: # aka sources
serviceMeshes:
- name: istio-sidecar
namespace: istio-system
others: # TBD
- name: "coroot"
# Either gRPC or HTTP not both
grpc:
url: localhost
port: 1234
http:
url: localhost
port: 4321
# Uncomment the following receivers according to your requirement.

# serviceMeshes:
# - name: istio-sidecar
# namespace: istio-system

# others:
# - name: nginx-inc-ingress-controller
# namespace: default

# - name: nginx-webserver

exporter:
grpc:
Expand Down
68 changes: 44 additions & 24 deletions sentryflow/pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,23 @@ import (

"github.com/spf13/viper"
"go.uber.org/zap"

"github.com/5GSEC/SentryFlow/pkg/util"
)

const (
DefaultConfigFilePath = "config/default.yaml"
SentryFlowDefaultFilterServerPort = 8081
)

type endpoint struct {
Url string `json:"url"`
Port uint16 `json:"port"`
}

type base struct {
Name string `json:"name,omitempty"`
// Todo: Do we really need both gRPC and http variants?
Grpc *endpoint `json:"grpc,omitempty"`
Http *endpoint `json:"http,omitempty"`
}

type serviceMesh struct {
type nameAndNamespace struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
Namespace string `json:"namespace,omitempty"`
}

type receivers struct {
ServiceMeshes []*serviceMesh `json:"serviceMeshes,omitempty"`
Others []*base `json:"others,omitempty"`
ServiceMeshes []*nameAndNamespace `json:"serviceMeshes,omitempty"`
Others []*nameAndNamespace `json:"other,omitempty"`
}

type envoyFilterConfig struct {
Expand All @@ -46,15 +36,26 @@ type server struct {
Port uint16 `json:"port"`
}

type nginxIngressConfig struct {
DeploymentName string `json:"deploymentName"`
ConfigMapName string `json:"configMapName"`
SentryFlowNjsConfigMapName string `json:"sentryFlowNjsConfigMapName"`
}

type filters struct {
Envoy *envoyFilterConfig `json:"envoy,omitempty"`
Server *server `json:"server,omitempty"`
Envoy *envoyFilterConfig `json:"envoy,omitempty"`
NginxIngress *nginxIngressConfig `json:"nginxIngress,omitempty"`
Server *server `json:"server,omitempty"`
}

type exporterConfig struct {
Grpc *server `json:"grpc"`
}

type Config struct {
Filters *filters `json:"filters"`
Receivers *receivers `json:"receivers"`
Exporter *base `json:"exporter"`
Filters *filters `json:"filters"`
Receivers *receivers `json:"receivers"`
Exporter *exporterConfig `json:"exporter"`
}

func (c *Config) validate() error {
Expand All @@ -73,9 +74,6 @@ func (c *Config) validate() error {
if c.Exporter.Grpc == nil {
return fmt.Errorf("no exporter's gRPC configuration provided")
}
if c.Exporter.Http != nil {
return fmt.Errorf("http exporter is not supported")
}

if c.Receivers == nil {
return fmt.Errorf("no receiver configuration provided")
Expand All @@ -90,6 +88,28 @@ func (c *Config) validate() error {
}
}

for _, other := range c.Receivers.Others {
if other.Name == "" {
return fmt.Errorf("no other receiver name provided")
}
if other.Name == util.NginxIncorporationIngressController {
if other.Namespace == "" {
return fmt.Errorf("no nginx-inc ingress controller namespace provided")
}
if c.Filters.NginxIngress == nil {
return fmt.Errorf("no nginx-inc ingress configuration provided")
}
if c.Filters.NginxIngress.DeploymentName == "" {
return fmt.Errorf("no nginx ingress deployment name provided")
}
if c.Filters.NginxIngress.ConfigMapName == "" {
return fmt.Errorf("no nginx ingress configmap name provided")
}
if c.Filters.NginxIngress.SentryFlowNjsConfigMapName == "" {
return fmt.Errorf("no sentryflow njs configmap name provided")
}
}
}
return nil
}

Expand Down
30 changes: 25 additions & 5 deletions sentryflow/pkg/core/sentryflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import (
"google.golang.org/grpc"
"istio.io/client-go/pkg/apis/extensions/v1alpha1"
networkingv1alpha3 "istio.io/client-go/pkg/apis/networking/v1alpha3"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand All @@ -34,6 +36,20 @@ type Manager struct {
ApiEvents chan *protobuf.APIEvent
}

func (m *Manager) areK8sReceivers(cfg *config.Config) bool {
if len(cfg.Receivers.ServiceMeshes) > 0 {
return true
}

for _, other := range cfg.Receivers.Others {
if other.Name == util.NginxIncorporationIngressController {
return true
}
}

return false
}

func Run(ctx context.Context, configFilePath string, kubeConfig string) {
mgr := &Manager{
Ctx: ctx,
Expand All @@ -50,12 +66,14 @@ func Run(ctx context.Context, configFilePath string, kubeConfig string) {
return
}

k8sClient, err := k8s.NewClient(registerAndGetScheme(), kubeConfig)
if err != nil {
mgr.Logger.Errorf("failed to create k8s client: %v", err)
return
if mgr.areK8sReceivers(cfg) {
k8sClient, err := k8s.NewClient(registerAndGetScheme(), kubeConfig)
if err != nil {
mgr.Logger.Errorf("failed to create k8s client: %v", err)
return
}
mgr.K8sClient = k8sClient
}
mgr.K8sClient = k8sClient

mgr.Wg.Add(1)
go func() {
Expand Down Expand Up @@ -95,6 +113,8 @@ func Run(ctx context.Context, configFilePath string, kubeConfig string) {
func registerAndGetScheme() *runtime.Scheme {
scheme := runtime.NewScheme()
utilruntime.Must(networkingv1alpha3.AddToScheme(scheme))
utilruntime.Must(corev1.AddToScheme(scheme))
utilruntime.Must(appsv1.AddToScheme(scheme))
utilruntime.Must(v1alpha1.AddToScheme(scheme))
return scheme
}
161 changes: 161 additions & 0 deletions sentryflow/pkg/receiver/other/nginx/nginxinc/nginx.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright 2024 Authors of SentryFlow

package nginxinc

import (
"context"
"fmt"
"strings"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/5GSEC/SentryFlow/pkg/config"
"github.com/5GSEC/SentryFlow/pkg/util"
)

func Start(ctx context.Context, cfg *config.Config, k8sClient client.Client) {
logger := util.LoggerFromCtx(ctx)

logger.Info("Starting nginx-incorporation ingress controller receiver")
if err := validateResources(ctx, cfg, k8sClient); err != nil {
// Todo(@anurag-rajawat): Log docs link for reference on how to configure this receiver properly.
logger.Errorf("%v. Stopped nginx-incorporation ingress controller receiver", err)
return
}
logger.Info("Started nginx-incorporation ingress controller receiver")

<-ctx.Done()
logger.Info("Shutting down nginx-incorporation ingress controller receiver")
logger.Info("Stopped nginx-incorporation ingress controller receiver")
}

func validateResources(ctx context.Context, cfg *config.Config, k8sClient client.Client) error {
ingressDeployNamespace := getIngressControllerDeploymentNamespace(cfg)
sentryFlowNjsCm := &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: cfg.Filters.NginxIngress.SentryFlowNjsConfigMapName,
Namespace: ingressDeployNamespace,
},
}

if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(sentryFlowNjsCm), sentryFlowNjsCm); err != nil {
return fmt.Errorf("failed to get sentryflow configmap: %w", err)
}

if err := validateIngressDeployAndConfigMap(ctx, cfg, k8sClient, ingressDeployNamespace); err != nil {
return err
}

return nil
}

func validateIngressDeployAndConfigMap(ctx context.Context, cfg *config.Config, k8sClient client.Client, ingressNamespace string) error {
ingressDeploy := &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
APIVersion: "apps/v1",
Kind: "Deployment",
},
ObjectMeta: metav1.ObjectMeta{
Name: cfg.Filters.NginxIngress.DeploymentName,
Namespace: ingressNamespace,
},
}
if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(ingressDeploy), ingressDeploy); err != nil {
return fmt.Errorf("failed to get nginx-incorporation ingress controller deployment: %w", err)
}

// Just check ingress controller deployment volume mount because if volume
// itself doesn't exist, the container will not start.
volumeMountFound := false
for _, container := range ingressDeploy.Spec.Template.Spec.Containers {
for _, volumeMount := range container.VolumeMounts {
// Volume-mount name could be different so only check mount-path.
if volumeMount.MountPath == "/etc/nginx/njs/sentryflow.js" {
volumeMountFound = true
}
}
}
if !volumeMountFound {
return fmt.Errorf("sentryflow-njs volume-mount not found")
}

ingressCm := &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: cfg.Filters.NginxIngress.ConfigMapName,
Namespace: ingressNamespace,
},
}
if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(ingressCm), ingressCm); err != nil {
return fmt.Errorf("failed to get nginx-incorporation ingress controller configmap: %w", err)
}

httpSnippets, exists := ingressCm.Data["http-snippets"]
if !exists {
return fmt.Errorf("sentryflow http-snippets not found in nginx-incorporation ingress configmap")
}
expectedHttpSnippets := `js_path "/etc/nginx/njs/";
subrequest_output_buffer_size 8k;
js_shared_dict_zone zone=apievents:1M timeout=300s evict;
js_import main from sentryflow.js;
`
if !strings.Contains(httpSnippets, expectedHttpSnippets) {
return fmt.Errorf("sentryflow http-snippets were not properly configured in nginx-incorporation ingress configmap")
}

locationSnippets, exists := ingressCm.Data["location-snippets"]
if !exists {
return fmt.Errorf("sentryflow location-snippets not found in nginx-incorporation ingress configmap")
}
expectedLocationSnippets := `js_body_filter main.requestHandler buffer_type=buffer;
mirror /mirror_request;
mirror_request_body on;
`
if !strings.Contains(locationSnippets, expectedLocationSnippets) {
return fmt.Errorf("sentryflow location-snippets were not properly configured in nginx-incorporation ingress configmap")
}

serverSnippets, exists := ingressCm.Data["server-snippets"]
if !exists {
return fmt.Errorf("sentryflow server-snippets not found in nginx-incorporation ingress configmap")
}
expectedServerSnippets := `location /mirror_request {
internal;
js_content main.dispatchHttpCall;
}
location /sentryflow {
internal;
proxy_method POST;
proxy_set_header accept "application/json";
proxy_set_header Content-Type "application/json";
}
`
// The server snippet might have different SentryFlow URL in `proxy_pass`
// directive. To avoid potential conflicts, check without that directive.
if !strings.ContainsAny(serverSnippets, expectedServerSnippets) {
return fmt.Errorf("sentryflow server-snippets were not properly configured in nginx-incorporation ingress configmap")
}

return nil
}

func getIngressControllerDeploymentNamespace(cfg *config.Config) string {
for _, other := range cfg.Receivers.Others {
switch other.Name {
case util.NginxIncorporationIngressController:
return other.Namespace
}
}
return ""
}
Loading

0 comments on commit b729367

Please sign in to comment.