add gitlab templates and docker

.github/workflows/build_push_dev.yml

@@ -0,0 +1,31 @@
+name: Build and push dev images
+
+on: workflow_dispatch
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Build and push scanners
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./docker//Dockerfile
+          push: true
+          tags: stefanfle/secobserve-scanners:dev

README.md

@@ -0,0 +1 @@
+# SecObserve_GitLab_Includes

SAST/bandit/action.yaml → actions/SAST/bandit/action.yaml

@@ -20,9 +20,11 @@ inputs:
 
 runs:
   using: 'docker'
-  image: "Dockerfile"
+  image: 'stefanfle/secobserve-scanners:dev'
+  entrypoint: 'entrypoints/entrypoint_bandit.sh'
   env:
-    INPUT_TARGET: ${{ inputs.target }}
-    INPUT_REPORT_NAME: ${{ inputs.report_name }}
-    INPUT_RUN_DIRECTORY: ${{ inputs.run_directory }}
-    INPUT_FURTHER_PARAMETERS: ${{ inputs.further_parameters }}
+    TARGET: ${{ inputs.target }}
+    REPORT_NAME: ${{ inputs.report_name }}
+    RUN_DIRECTORY: ${{ inputs.run_directory }}
+    FURTHER_PARAMETERS: ${{ inputs.further_parameters }}
+    WORKSPACE: ${{ vars.GITHUB_WORKSPACE }}

docker/Dockerfile

@@ -0,0 +1,74 @@
+# Python build stage
+FROM python:3.11.1-alpine as python-build-stage
+
+# Install gcc to be able to compile wheels for python packages
+RUN apk add --no-cache gcc
+
+# Generate wheels for Python packages
+WORKDIR /usr/local
+COPY docker/requirements.txt .
+RUN pip wheel --wheel-dir /usr/src/app/wheels -r ./requirements.txt
+
+# Go build stage for KICS
+FROM golang:1.20.0-alpine as go-build-stage
+
+ARG KICS_VERSION=1.6.8
+
+# Install kics from GitHub
+WORKDIR /usr/local/kics
+RUN apk add --no-cache build-base
+RUN wget --no-verbose https://github.com/Checkmarx/kics/archive/refs/tags/v${KICS_VERSION}.tar.gz -O - | tar -zxf - \
+    && cd kics-${KICS_VERSION} \
+    && go build -o ./bin/kics cmd/console/main.go
+
+# Python run stage
+FROM python:3.11.1-alpine as python-run-stage
+
+ARG GITLEAKS_VERSION=8.15.3
+ARG GRYPE_VERSION=0.55.0
+ARG KICS_VERSION=1.6.8
+ARG TRIVY_VERSION=0.36.1
+
+# All absolute dir copies ignore workdir instruction. All relative dir copies are wrt to the workdir instruction
+# copy python dependency wheels from python-build-stage
+COPY --from=python-build-stage /usr/src/app/wheels  /wheels/
+# use wheels to install python dependencies
+RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \
+	&& rm -rf /wheels/ && rm -rf /tmp
+
+# copy and install precompiled DrHeader library and rules
+COPY docker/drheader/drheader-1.7.0-py2.py3-none-any.whl docker/drheader/rules.yml ./
+RUN pip install --no-cache-dir ./drheader-1.7.0-py2.py3-none-any.whl
+
+# install GitLeaks from Github
+WORKDIR /usr/local/gitleaks
+RUN wget --no-verbose https://github.com/zricethezav/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz -O - | tar -zxf -
+
+# install Grype from Github
+WORKDIR /usr/local/grype
+RUN wget --no-verbose https://github.com/anchore/grype/releases/download/v${GRYPE_VERSION}/grype_${GRYPE_VERSION}_linux_amd64.tar.gz -O - | tar -zxf -
+
+# Copy kics from go build stage
+WORKDIR /usr/local/kics
+COPY --from=go-build-stage /usr/local/kics/kics-${KICS_VERSION}/bin ./bin/
+COPY --from=go-build-stage /usr/local/kics/kics-${KICS_VERSION}/assets/queries ./queries/
+
+# Install trivy from GitHub
+WORKDIR /usr/local/trivy
+RUN wget --no-verbose https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz -O - | tar -zxf -
+
+# eslint needs npm
+# gitleaks needs git
+# trivy needs docker
+RUN apk add --no-cache npm git docker-cli
+
+# Install importer
+WORKDIR /usr/local/importer
+COPY importer/ ./
+
+# Copy entrypoints and set PATH
+WORKDIR /entrypoints
+COPY ./docker/entrypoints/* .
+ENV PATH="/usr/local/gitleaks:/usr/local/grype:/usr/local/kics/bin:/usr/local/trivy:/usr/local/importer/bin:$PATH"
+
+RUN mkdir -p -m a=rwx /tmp

docker/drheader/rules.yml

@@ -0,0 +1,68 @@
+Headers:
+    Content-Security-Policy:
+        Required: True
+        Must-Avoid:
+            - unsafe-inline
+            - unsafe-eval
+        Directives:
+            default-src:
+                Required: True
+                Value-One-Of:
+                    - none
+                    - self
+    Cross-Origin-Embedder-Policy:
+        Required: True
+        Value: require-corp
+    Cross-Origin-Opener-Policy:
+        Required: True
+        Value: same-origin
+    Cross-Origin-Resource-Policy:
+        Required: True
+        Value: same-site
+    Pragma:
+        Required: False
+    Referrer-Policy:
+        Required: True
+        Value-One-Of:
+            - strict-origin
+            - strict-origin-when-cross-origin
+            - no-referrer
+            - no-referrer, strict-origin-when-cross-origin
+    Server:
+        Required: Optional
+        Value:
+            - undisclosed
+    Set-Cookie:
+        Required: Optional
+        Must-Contain:
+            - HttpOnly
+            - Secure
+            - SameSite=Strict
+    Strict-Transport-Security:
+        Required: True
+        Value:
+            - max-age=31536000
+            - includeSubDomains
+            - preload
+    User-Agent:
+        Required: False
+    X-AspNet-Version:
+        Required: False
+    X-Client-IP:
+        Required: False
+    X-Content-Type-Options:
+        Required: True
+        Value: nosniff
+    X-Forwarded-For:
+        Required: False
+    X-Frame-Options:
+        Required: True
+        Value-One-Of:
+            - DENY
+            - SAMEORIGIN
+    X-Generator:
+        Required: False
+    X-Powered-By:
+        Required: False
+    X-XSS-Protection:
+        Required: False

docker/entrypoints/entrypoint_bandit.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+cd "$RUN_DIRECTORY"
+bandit $FURTHER_PARAMETERS --format sarif --output "$WORKSPACE/$REPORT_NAME" --exit-zero --recursive "$TARGET"
+cd "$WORKSPACE"
+
+exit 0

docker/requirements.txt

@@ -0,0 +1,20 @@
+# Bandit
+# ----------------------------------------------------------------
+bandit==1.7.4  # https://github.com/PyCQA/bandit
+bandit-sarif-formatter==1.1.1  # https://github.com/microsoft/bandit-sarif-formatter
+
+# Checkov
+# ----------------------------------------------------------------
+checkov==2.2.335  # https://github.com/bridgecrewio/checkov
+
+# Semgrep
+# ----------------------------------------------------------------
+semgrep==1.9.0  # https://github.com/returntocorp/semgrep
+
+# SSLyze
+# ----------------------------------------------------------------
+sslyze==5.1.1  # https://github.com/nabla-c0d3/sslyze
+
+# Importer
+# ----------------------------------------------------------------
+requests==2.28.2

docker_zap/Dockerfile

@@ -0,0 +1,8 @@
+# Python run stage
+FROM owasp/zap2docker-stable
+
+# Install importer
+COPY importer/ /usr/local/importer/
+RUN pip install --no-cache-dir -r /usr/local/importer/requirements.txt
+
+ENV PATH="/usr/local/importer/bin:$PATH"

importer/.dockerignore

@@ -0,0 +1,11 @@
+.git
+.github
+coverage_data
+data
+docker
+**/__pycache__
+env-file.txt
+dd_import_stefanf.egg-info
+dd_import.egg-info
+distproject.toml
+setup.cfg

importer/.gitignore

@@ -0,0 +1,7 @@
+*/__pycache__/*
+env-file.txt
+data/
+coverage_data/
+dd_import_stefanf.egg-info
+dd_import.egg-info
+dist

importer/bin/file_upload_observations.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+export PYTHONPATH="${PYTHONPATH}:/usr/local/importer"
+python -m importer.file_upload_observations

importer/importer/environment.py

@@ -0,0 +1,55 @@
+import os
+
+
+class Environment:
+    def __init__(self):
+        self.api_base_url = os.getenv("SO_API_BASE_URL")
+        self.api_token = os.getenv("SO_API_TOKEN")
+        self.product_name = os.getenv("SO_PRODUCT_NAME")
+        self.file_name = os.getenv("SO_FILE_NAME", None)
+        self.parser_name = os.getenv("SO_PARSER_NAME", None)
+        self.api_configuration_name = os.getenv("SO_API_CONFIGURATION_NAME", None)
+        self.service = os.getenv("SO_ORIGIN_SERVICE", None)
+        self.docker_image_name_tag = os.getenv("SO_ORIGIN_DOCKER_IMAGE_NAME_TAG", None)
+        self.endpoint_url = os.getenv("SO_ORIGIN_ENDPOINT_URL", None)
+
+    def check_environment_file_upload(self):
+        error_string = self.check_environment_common()
+        if self.file_name is None:
+            if error_string != "":
+                error_string = error_string + " / "
+            error_string = error_string + "SO_FILE_NAME is missing"
+        if self.parser_name is None:
+            if error_string != "":
+                error_string = error_string + " / "
+            error_string = error_string + "SO_PARSER_NAME is missing"
+
+        if len(error_string) > 0:
+            raise Exception(error_string)
+
+        print("SO_API_BASE_URL:                 ", self.api_base_url)
+        print("SO_PRODUCT_NAME:                 ", self.product_name)
+        print("SO_FILE_NAME:                    ", self.file_name)
+        print("SO_PARSER_NAME:                  ", self.parser_name)
+        if self.service:
+            print("SO_ORIGIN_SERVICE:               ", self.service)
+        if self.docker_image_name_tag:
+            print("SO_ORIGIN_DOCKER_IMAGE_NAME_TAG: ", self.docker_image_name_tag)
+        if self.endpoint_url:
+            print("SO_ORIGIN_ENDPOINT_URL:          ", self.endpoint_url)
+        print("")
+
+    def check_environment_common(self):
+        error_string = ""
+        if self.api_base_url is None:
+            error_string = "SO_API_BASE_URL is missing"
+        if self.api_token is None:
+            if error_string != "":
+                error_string = error_string + " / "
+            error_string = error_string + "SO_API_TOKEN is missing"
+        if self.product_name is None:
+            if error_string != "":
+                error_string = error_string + " / "
+            error_string = error_string + "SO_PRODUCT_NAME is missing"
+
+        return error_string

importer/importer/file_upload_observations.py

@@ -0,0 +1,17 @@
+from importer.secobserve_api import Api
+from importer.environment import Environment
+
+
+def file_upload_observations():
+    try:
+        environment = Environment()
+        environment.check_environment_file_upload()
+        api = Api()
+        api.file_upload_observations()
+    except Exception as e:
+        print(f"{e.__class__.__name__}: {str(e)}")
+        exit(1)
+
+
+if __name__ == "__main__":
+    file_upload_observations()

importer/importer/secobserve_api.py

@@ -0,0 +1,54 @@
+import requests
+from http import HTTPStatus
+
+from importer.environment import Environment
+
+
+class Api:
+    def __init__(self):
+        self.environment = Environment()
+        self.headers = {
+            "accept": "application/json",
+            "Content-type": "application/json",
+            "Authorization": "APIToken " + self.environment.api_token,
+        }
+        self.headers_multipart = {
+            "accept": "application/json",
+            # "Content-Type": "multipart/form-data",
+            "Authorization": "APIToken " + self.environment.api_token
+        }
+        self.file_upload_url = (
+            self.environment.api_base_url
+            + "/api/import/file_upload_observations_by_name/"
+        )
+
+    def file_upload_observations(self):
+        payload = {
+            "product_name": self.environment.product_name,
+            "parser_name": self.environment.parser_name,
+        }
+        if self.environment.service is not None:
+            payload["service"] = self.environment.service
+        if self.environment.docker_image_name_tag is not None:
+            payload["docker_image_name_tag"] = self.environment.docker_image_name_tag
+        if self.environment.endpoint_url is not None:
+            payload["endpoint_url"] = self.environment.endpoint_url
+
+        with open(self.environment.file_name, "r") as file:
+            file.seek(0)
+            files = {
+                "file": (
+                    self.environment.file_name,
+                    file,
+                    "application/json",
+                )
+            }
+            response = requests.post(
+                self.file_upload_url,
+                headers=self.headers_multipart,
+                data=payload,
+                files=files,
+            )
+            response.raise_for_status()
+
+            print(response.json())

importer/requirements.txt

@@ -0,0 +1 @@
+requests==2.28.2