From bc3382c307eff7a58f4547494a496572976e9893 Mon Sep 17 00:00:00 2001
From: Joachim Metz <joachim.metz@gmail.com>
Date: Sun, 5 May 2024 06:52:06 +0200
Subject: [PATCH] Changes for end-to-end tests

---
 config/end_to_end/nsrlsvr.Dockerfile       | 32 ++++++++++
 config/end_to_end/run_tests_with_docker.sh | 15 -----
 config/jenkins/run_end_to_end_tests.sh     |  8 ++-
 config/jenkins/run_nsrlsvr.sh              | 69 ++++++++++++++++++++
 config/linux/ubuntu_install_nsrlsvr.sh     | 74 ----------------------
 config/linux/ubuntu_install_opensearch.sh  | 36 -----------
 6 files changed, 107 insertions(+), 127 deletions(-)
 create mode 100644 config/end_to_end/nsrlsvr.Dockerfile
 create mode 100755 config/jenkins/run_nsrlsvr.sh
 delete mode 100755 config/linux/ubuntu_install_nsrlsvr.sh
 delete mode 100755 config/linux/ubuntu_install_opensearch.sh

diff --git a/config/end_to_end/nsrlsvr.Dockerfile b/config/end_to_end/nsrlsvr.Dockerfile
new file mode 100644
index 0000000000..7f5a8cbcf9
--- /dev/null
+++ b/config/end_to_end/nsrlsvr.Dockerfile
@@ -0,0 +1,32 @@
+FROM ubuntu:jammy
+MAINTAINER Log2Timeline <log2timeline-dev@go
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Combining the apt-get commands into a single run reduces the size of the resulting image.
+# The apt-get installations below are interdependent and need to be done in sequence.
+RUN apt-get -y update && \
+    apt-get -y install apt-transport-https apt-utils && \
+    apt-get -y install libterm-readline-gnu-perl software-properties-common && \
+    apt-get -y install locales
+
+# Set terminal to UTF-8 by default.
+RUN locale-gen en_US.UTF-8
+RUN update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+
+# Install nsrlsvr.
+RUN add-apt-repository ppa:gift/dev -y && \
+    apt-get update -q && \
+    apt-get install -y nsrlsvr-server
+
+# Initialize nsrlsvr hashes.txt file.
+RUN mkdir -p /var/share/nsrlsvr && \
+    mkdir -p /usr/share/nsrlsvr && \
+    touch /usr/share/nsrlsvr/hashes.txt
+
+WORKDIR /home/test/
+
+# Clean up apt-get cache files.
+RUN apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/*
diff --git a/config/end_to_end/run_tests_with_docker.sh b/config/end_to_end/run_tests_with_docker.sh
index 774f2784d4..cdfc222476 100755
--- a/config/end_to_end/run_tests_with_docker.sh
+++ b/config/end_to_end/run_tests_with_docker.sh
@@ -73,25 +73,10 @@ do
 
 	# TODO: move custom test setup and teardown scripts to configuration parameter?
 
-	if [[ ${OUTPUT_FORMAT} == "opensearch" ]] || [[ ${OUTPUT_FORMAT} == "opensearch_ts" ]];
-	then
-		# Install OpenSearch and give it 3 minutes to start-up before running the output end-to-end test.
-		COMMAND="./config/linux/ubuntu_install_opensearch.sh && sleep 3m && ${COMMAND}";
-	fi
-
 	if [[ ${TEST_NAME} == "acserver-mounted" ]];
 	then
 		COMMAND="mkdir -p /mnt/acserver_mount && mount -o ro,noload,noacl,loop,offset=1048576 /sources/acserver.dd /mnt/acserver_mount && ./tests/end-to-end.py --config /config/${TEST_NAME}.ini --references-directory test_data/end_to_end --results-directory plaso-out --sources-directory /mnt --scripts-directory plaso/scripts && umount /mnt/acserver_mount && rmdir /mnt/acserver_mount";
 
-	elif [[ ${TEST_NAME} == *\-nsrlsvr ]];
-	then
-		# Install nsrlsvr and give it 3 minutes to start-up before running the output end-to-end test.
-		COMMAND="./config/linux/ubuntu_install_nsrlsvr.sh && sleep 3m && ${COMMAND}";
-
-	elif [[ ${TEST_NAME} == *\-redis ]];
-	then
-		# TODO: add support for Redis tests
-		continue;
 	fi
 	echo "Running ${TEST_CASE} end-to-end test: ${TEST_NAME}";
 
diff --git a/config/jenkins/run_end_to_end_tests.sh b/config/jenkins/run_end_to_end_tests.sh
index 4069331fd3..d31ef7ead0 100755
--- a/config/jenkins/run_end_to_end_tests.sh
+++ b/config/jenkins/run_end_to_end_tests.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Script to run end-to-end tests on a Linux Jenkins instance with Docker.
+# Script to run end-to-end tests on an Ubuntu Jenkins instance with Docker.
 
 # Fail on error.
 set -e
@@ -33,7 +33,11 @@ docker run log2timeline/plaso ./utils/check_dependencies.py;
 
 COMMAND="./tests/end-to-end.py --config /config/${CONFIGURATION_NAME}.ini --references-directory test_data/end_to_end --results-directory /home/test/plaso/plaso-out --sources-directory /sources --scripts-directory plaso/scripts";
 
-if test ${CONFIGURATION_NAME} = "output_opensearch";
+if test ${CONFIGURATION_NAME} = "psort-studentpc1-nsrlsvr";
+then
+	DOCKER_NETWORK="--network=nsrlsvr-network";
+
+elif test ${CONFIGURATION_NAME} = "output_opensearch" || test ${CONFIGURATION_NAME} = "output_opensearch_ts";
 then
 	DOCKER_NETWORK="--network=opensearch-network";
 
diff --git a/config/jenkins/run_nsrlsvr.sh b/config/jenkins/run_nsrlsvr.sh
new file mode 100755
index 0000000000..c0aaf78bda
--- /dev/null
+++ b/config/jenkins/run_nsrlsvr.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+#
+# Script to run nsrlsvr on an Ubuntu Jenkins instance with Docker.
+
+AUXILIARY_DATA_PATH="/media/auxiliary";
+
+# Exit on error.
+set -e
+
+sudo apt-get install -y curl unzip
+
+if [ -f "${AUXILIARY_DATA_PATH}/nsrlsvr/NSRLFile.txt" ];
+then
+	# Note that NSRLFile.txt is approximate 4 GiB in size.
+	cp -f "${AUXILIARY_DATA_PATH}/nsrlsvr/NSRLFile.txt" .
+fi
+
+if [ ! -f NSRLFile.txt ];
+then
+	if [ -f "${AUXILIARY_DATA_PATH}/nsrlsvr/rds_modernm.zip" ];
+	then
+		# Note that this is an older rds_modernm.zip that is approximate 2 GiB in size.
+		cp -f "${AUXILIARY_DATA_PATH}/nsrlsvr/rds_modernm.zip" .
+	fi
+
+	if [ ! -f rds_modernm.zip ];
+	then
+		# Download the minimum modern RDS hash set.
+		# Note that rds_modernm.zip is approximate 18 GiB in size.
+		curl -o rds_modernm.zip https://s3.amazonaws.com/rds.nsrl.nist.gov/RDS/rds_2024.03.1/RDS_2024.03.1_modern_minimal.zip
+	fi
+
+	if [ ! -f rds_modernm.zip ];
+	then
+		echo "Missing: rds_modernm.zip";
+
+		exit 1
+	fi
+
+	unzip -x rds_modernm.zip rds_modernm/NSRLFile.txt
+
+	mv rds_modernm/NSRLFile.txt .
+fi
+
+if [ ! -f NSRLFile.txt ];
+then
+	echo "Missing: NSRLFile.txt";
+
+	exit 1
+fi
+
+docker build -f nsrlsvr.Dockerfile --force-rm --no-cache -t log2timeline/nsrlsvr . ;
+
+# Update the nsrlsvr hashes.txt file from NSRLFile.txt
+docker run -v "${AUXILIARY_DATA_PATH}:/data:z" log2timeline/nsrlsvr /bin/bash -c "/usr/bin/python3 /usr/bin/nsrlupdate /data/NSRLFile.txt";
+
+# Preserver the intermediate container so we don't have to rebuild hashes.txt
+docker commit `docker ps -lq` | cut -c8- > nsrlsvr-container
+
+# Run nsrlsvr self tests
+docker run `cat nsrlsvr-container` /bin/bash -c "time /usr/bin/nsrlsvr --dry-run";
+
+# Run nsrlsvr listening on port 9120
+docker run -d --name=nsrlsvr --network=nsrlsvr-network -p 127.0.0.1:9120:9120 `cat nsrlsvr-container` /bin/bash -c "/usr/bin/nsrlsvr -p 9120";
+
+# Give nsrlsvr 1 minute to load the hashes and be ready for action
+# `time nsrlsvr --dry-run' indicates this takes about 30 seconds
+sleep 60
+
diff --git a/config/linux/ubuntu_install_nsrlsvr.sh b/config/linux/ubuntu_install_nsrlsvr.sh
deleted file mode 100755
index 1327789a51..0000000000
--- a/config/linux/ubuntu_install_nsrlsvr.sh
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/usr/bin/env bash
-#
-# Script to install nsrlsvr on Ubuntu from the GIFT PPA.
-
-AUXILIARY_DATA_PATH="/media/auxiliary";
-
-# Exit on error.
-set -e
-
-# Install and configure nsrlsvr
-sudo add-apt-repository ppa:gift/dev -y
-sudo apt-get update -q
-sudo apt-get install -y curl net-tools nsrlsvr-server unzip
-
-sudo mkdir -p /var/share/nsrlsvr
-
-if [ -f "${AUXILIARY_DATA_PATH}/nsrlsvr/hashes.txt" ];
-then
-	cp -f "${AUXILIARY_DATA_PATH}/nsrlsvr/hashes.txt" /var/share/nsrlsvr;
-fi
-
-# Have nsrlupdate generate /var/share/nsrlsvr/hashes.txt
-if [ ! -f /var/share/nsrlsvr/hashes.txt ];
-then
-	if [ -f "${AUXILIARY_DATA_PATH}/nsrlsvr/NSRLFile.txt" ];
-	then
-		cp -f "${AUXILIARY_DATA_PATH}/nsrlsvr/NSRLFile.txt" .
-	fi
-
-	if [ ! -f NSRLFile.txt ];
-	then
-		if [ -f "${AUXILIARY_DATA_PATH}/nsrlsvr/rds_modernm.zip" ];
-		then
-			cp -f "${AUXILIARY_DATA_PATH}/nsrlsvr/rds_modernm.zip" .
-		fi
-
-		if [ ! -f rds_modernm.zip ];
-		then
-			# Download the minimum RDS hash set.
-			# Note that rds_modernm.zip is approximate 2 GiB in size.
-			curl -o rds_modernm.zip https://s3.amazonaws.com/rds.nsrl.nist.gov/RDS/rds_2024.03.1/RDS_2024.03.1_modern_minimal.zip
-		fi
-
-		if [ ! -f rds_modernm.zip ];
-		then
-			echo "Missing: rds_modernm.zip";
-
-			exit 1
-		fi
-		# Note that NSRLFile.txt is approximate 4 GiB in size.
-		unzip -x rds_modernm.zip rds_modernm/NSRLFile.txt
-
-		mv rds_modernm/NSRLFile.txt .
-	fi
-
-	if [ ! -f NSRLFile.txt ];
-	then
-		echo "Missing: NSRLFile.txt";
-
-		exit 1
-	fi
-
-	# Build the nsrlsvr hashes.txt file
-	sudo mkdir -p /usr/share/nsrlsvr
-	sudo touch /usr/share/nsrlsvr/hashes.txt
-	sudo /usr/bin/python3 /usr/bin/nsrlupdate NSRLFile.txt
-fi
-
-# For the sake of verbosity have nsrlsvr test its set up first
-time sudo /usr/bin/nsrlsvr --dry-run
-
-# Run nsrlsvr listening on port 9120
-sudo /usr/bin/nsrlsvr -p 9120
-
diff --git a/config/linux/ubuntu_install_opensearch.sh b/config/linux/ubuntu_install_opensearch.sh
deleted file mode 100755
index 8568e0c9c5..0000000000
--- a/config/linux/ubuntu_install_opensearch.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-#
-# Script to install OpenSearch on Ubuntu
-
-# Exit on error.
-set -e
-
-OPENSEARCH_VERSION="1.2.2";
-
-sudo apt-get update
-sudo apt-get install -y wget
-
-# TODO: update /etc/sysctl.conf
-# vm.max_map_count=262144
-
-adduser opensearch
-
-# Download OpenSearch
-
-wget -q https://artifacts.opensearch.org/releases/bundle/opensearch/${OPENSEARCH_VERSION}/opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz
-
-# Install OpenSearch
-
-tar xfv ${PWD}/opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz
-
-# Install the OpenSearch Python bindings
-
-sudo apt-get install -y python3-opensearch
-
-# Start OpenSearch
-
-cd opensearch-${OPENSEARCH_VERSION}
-
-echo "Starting OpenSearch";
-
-su -c './opensearch-tar-install.sh -Eplugins.security.disabled=true' opensearch &