Add data ingestion mechanism

.github/workflows/data_ingestion.yml

@@ -1,13 +1,38 @@
 name: Data ingestion
 on: ["push", "workflow_dispatch"]
 jobs:
-  test:
+  data_ingestion:
     environment: glvd-data-ingestion
     name: data ingestion
     runs-on: ubuntu-latest
+
+    env:
+      PGHOST: "${{ vars.PGHOST }}"
+      PGPORT: ${{ vars.PGPORT }}
+      PGUSER: ${{ vars.PGUSER }}
+      PGPASSWORD: ${{ secrets.PGPASSWORD }}
+      PGDATABASE: ${{ vars.PGDATABASE }}
+
     steps:
-      - run: |
-          echo "${{ vars.PGDATABASE }}"
-          echo "${{ vars.PGHOST }}"
-          echo "${{ vars.PGPORT }}"
-          echo "${{ vars.PGUSER }}"
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Build image for data ingestion container
+      id: build
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: container/Containerfile
+        load: true
+
+    - name: Run modified glvd container for data ingestion
+      run: >
+        docker run
+        --network host
+        --rm
+        --env PGHOST="$PGHOST"
+        --env PGPORT="$PGPORT"
+        --env PGUSER="$PGUSER"
+        --env PGPASSWORD="$PGPASSWORD"
+        --env PGDATABASE="$PGDATABASE"
+        ${{ steps.build.outputs.imageid }}

.gitignore

@@ -0,0 +1 @@
+container/tmp/*

container/Containerfile

@@ -0,0 +1,19 @@
+# Base Image
+FROM ghcr.io/gardenlinux/glvd:edge
+
+# Ru
+RUN mkdir -p /opt/glvd-data-ingestion/ingest-debsrc && \
+    mkdir -p /opt/glvd-data-ingestion/ingest-debsec
+
+# Copy temporary build files for container to the container
+COPY conf/ /opt/glvd-data-ingestion/ingest-debsrc
+
+# Copy ingestion scripts into the container
+Copy src/data_ingestion.sh /usr/local/bin/data_ingestion
+
+# Run some commands
+RUN APT_CONFIG=/opt/glvd-data-ingestion/ingest-debsrc/apt.conf apt-get update
+
+# Adjust entrypoint and command
+ENTRYPOINT /bin/bash
+CMD /usr/local/bin/data_ingestion

container/conf/apt/apt.conf

@@ -0,0 +1,9 @@
+Acquire::Retries "3";
+Dir "/opt/glvd-data-ingestion/ingest-debsrc/";
+Dir::Etc::parts "/dev/null";
+Dir::Etc::Preferences "/dev/null";
+Dir::Etc::PreferencesParts "/dev/null";
+Dir::Etc::sourcelist "/opt/glvd-data-ingestion/ingest-debsrc/sources.list";
+Dir::Etc::sourceparts "/dev/null";
+Dir::Etc::trusted "/usr/share/keyrings/debian-archive-keyring.gpg";
+Dir::State "/opt/glvd-data-ingestion/ingest-debsrc/";

container/conf/apt/sources.list

@@ -0,0 +1,7 @@
+deb-src https://deb.debian.org/debian bookworm main non-free-firmware
+deb-src https://deb.debian.org/debian bookworm-updates main non-free-firmware
+deb-src https://deb.debian.org/debian bookworm-proposed-updates main non-free-firmware
+deb-src https://deb.debian.org/debian bookworm-backports main non-free-firmware
+deb-src https://deb.debian.org/debian-security bookworm-security main non-free-firmware
+deb-src https://deb.debian.org/debian trixie main non-free-firmware
+deb-src [trusted=yes] https://repo.gardenlinux.io/gardenlinux 1337.0 main

src/data_ingestion.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# ---------
+# Ingest NVD data
+#glvd-data ingest-nvd
+
+# ---------
+# Ingest CVE lists
+glvd-data ingest-debsec debian /opt/glvd/ingest-debsec/debian
+glvd-data ingest-debsec gardenlinux /opt/glvd/ingest-debsec/gardenlinux
+
+# ---------
+# Ingest Debian sources
+glvd-data ingest-debsrc debian trixie /mnt/ingest-debsrc/lists/deb.debian.org_debian_dists_trixie_main_source_Sources
+glvd-data ingest-debsrc gardenlinux 1336.0 /mnt/ingest-debsrc/lists/repo.gardenlinux.io_gardenlinux_dists_1337.0_main_source_Sources

src/prepare_ingestion_container.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Preparation
+apt update
+apt install -y wget
+
+# Data Ingestion: Deb Sec
+mkdir -p container/tmp/ingest-debsec/debian/CVE
+mkdir -p container/tmp/ingest-debsec/gardenlinux/CVE
+wget https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/CVE/list?ref_type=heads -O container/tmp/ingest-debsec/debian/CVE/list
+cp -a data/CVE/list container/tmp/ingest-debsec/gardenlinux/CVE/list
+
+# Data Ingestion: Deb Src
+mkdir -p container/tmp/ingest-debsrc/
+cp -a container/conf/apt/* container/tmp/ingest-debsrc/

tmp/ingest-debsec/gardenlinux/CVE/list

@@ -0,0 +1,2 @@
+CVE-2023-6977 (This vulnerability enables malicious users to read sensitive files on  ...)
+	TODO: check

tmp/ingest-debsrc/apt.conf

@@ -0,0 +1,9 @@
+Acquire::Retries "3";
+Dir "/opt/glvd-data-ingestion/ingest-debsrc/";
+Dir::Etc::parts "/dev/null";
+Dir::Etc::Preferences "/dev/null";
+Dir::Etc::PreferencesParts "/dev/null";
+Dir::Etc::sourcelist "/opt/glvd-data-ingestion/ingest-debsrc/sources.list";
+Dir::Etc::sourceparts "/dev/null";
+Dir::Etc::trusted "/usr/share/keyrings/debian-archive-keyring.gpg";
+Dir::State "/opt/glvd-data-ingestion/ingest-debsrc/";

tmp/ingest-debsrc/sources.list

@@ -0,0 +1,7 @@
+deb-src https://deb.debian.org/debian bookworm main non-free-firmware
+deb-src https://deb.debian.org/debian bookworm-updates main non-free-firmware
+deb-src https://deb.debian.org/debian bookworm-proposed-updates main non-free-firmware
+deb-src https://deb.debian.org/debian bookworm-backports main non-free-firmware
+deb-src https://deb.debian.org/debian-security bookworm-security main non-free-firmware
+deb-src https://deb.debian.org/debian trixie main non-free-firmware
+deb-src [trusted=yes] https://repo.gardenlinux.io/gardenlinux today main