From a6b4e21738620f7680c8cd4f075e1539461ae890 Mon Sep 17 00:00:00 2001
From: Joachim Metz <joachim.metz@gmail.com>
Date: Sun, 5 May 2024 07:48:37 +0200
Subject: [PATCH] Changes for end-to-end tests

---
 config/end_to_end/nsrlsvr.Dockerfile       |  2 +-
 config/end_to_end/run_tests_with_docker.sh | 15 -----
 config/jenkins/run_end_to_end_tests.sh     |  8 ++-
 config/jenkins/run_nsrlsvr.sh              | 69 ++++++++++++++++++++++
 config/linux/ubuntu_install_opensearch.sh  | 36 -----------
 5 files changed, 76 insertions(+), 54 deletions(-)
 create mode 100755 config/jenkins/run_nsrlsvr.sh
 delete mode 100755 config/linux/ubuntu_install_opensearch.sh

diff --git a/config/end_to_end/nsrlsvr.Dockerfile b/config/end_to_end/nsrlsvr.Dockerfile
index 88a8d37cf8..7f5a8cbcf9 100644
--- a/config/end_to_end/nsrlsvr.Dockerfile
+++ b/config/end_to_end/nsrlsvr.Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:noble
+FROM ubuntu:jammy
 MAINTAINER Log2Timeline <log2timeline-dev@go
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/config/end_to_end/run_tests_with_docker.sh b/config/end_to_end/run_tests_with_docker.sh
index 774f2784d4..cdfc222476 100755
--- a/config/end_to_end/run_tests_with_docker.sh
+++ b/config/end_to_end/run_tests_with_docker.sh
@@ -73,25 +73,10 @@ do
 
 	# TODO: move custom test setup and teardown scripts to configuration parameter?
 
-	if [[ ${OUTPUT_FORMAT} == "opensearch" ]] || [[ ${OUTPUT_FORMAT} == "opensearch_ts" ]];
-	then
-		# Install OpenSearch and give it 3 minutes to start-up before running the output end-to-end test.
-		COMMAND="./config/linux/ubuntu_install_opensearch.sh && sleep 3m && ${COMMAND}";
-	fi
-
 	if [[ ${TEST_NAME} == "acserver-mounted" ]];
 	then
 		COMMAND="mkdir -p /mnt/acserver_mount && mount -o ro,noload,noacl,loop,offset=1048576 /sources/acserver.dd /mnt/acserver_mount && ./tests/end-to-end.py --config /config/${TEST_NAME}.ini --references-directory test_data/end_to_end --results-directory plaso-out --sources-directory /mnt --scripts-directory plaso/scripts && umount /mnt/acserver_mount && rmdir /mnt/acserver_mount";
 
-	elif [[ ${TEST_NAME} == *\-nsrlsvr ]];
-	then
-		# Install nsrlsvr and give it 3 minutes to start-up before running the output end-to-end test.
-		COMMAND="./config/linux/ubuntu_install_nsrlsvr.sh && sleep 3m && ${COMMAND}";
-
-	elif [[ ${TEST_NAME} == *\-redis ]];
-	then
-		# TODO: add support for Redis tests
-		continue;
 	fi
 	echo "Running ${TEST_CASE} end-to-end test: ${TEST_NAME}";
 
diff --git a/config/jenkins/run_end_to_end_tests.sh b/config/jenkins/run_end_to_end_tests.sh
index 4069331fd3..d31ef7ead0 100755
--- a/config/jenkins/run_end_to_end_tests.sh
+++ b/config/jenkins/run_end_to_end_tests.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Script to run end-to-end tests on a Linux Jenkins instance with Docker.
+# Script to run end-to-end tests on an Ubuntu Jenkins instance with Docker.
 
 # Fail on error.
 set -e
@@ -33,7 +33,11 @@ docker run log2timeline/plaso ./utils/check_dependencies.py;
 
 COMMAND="./tests/end-to-end.py --config /config/${CONFIGURATION_NAME}.ini --references-directory test_data/end_to_end --results-directory /home/test/plaso/plaso-out --sources-directory /sources --scripts-directory plaso/scripts";
 
-if test ${CONFIGURATION_NAME} = "output_opensearch";
+if test ${CONFIGURATION_NAME} = "psort-studentpc1-nsrlsvr";
+then
+	DOCKER_NETWORK="--network=nsrlsvr-network";
+
+elif test ${CONFIGURATION_NAME} = "output_opensearch" || test ${CONFIGURATION_NAME} = "output_opensearch_ts";
 then
 	DOCKER_NETWORK="--network=opensearch-network";
 
diff --git a/config/jenkins/run_nsrlsvr.sh b/config/jenkins/run_nsrlsvr.sh
new file mode 100755
index 0000000000..c0aaf78bda
--- /dev/null
+++ b/config/jenkins/run_nsrlsvr.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+#
+# Script to run nsrlsvr on an Ubuntu Jenkins instance with Docker.
+
+AUXILIARY_DATA_PATH="/media/auxiliary";
+
+# Exit on error.
+set -e
+
+sudo apt-get install -y curl unzip
+
+if [ -f "${AUXILIARY_DATA_PATH}/nsrlsvr/NSRLFile.txt" ];
+then
+	# Note that NSRLFile.txt is approximate 4 GiB in size.
+	cp -f "${AUXILIARY_DATA_PATH}/nsrlsvr/NSRLFile.txt" .
+fi
+
+if [ ! -f NSRLFile.txt ];
+then
+	if [ -f "${AUXILIARY_DATA_PATH}/nsrlsvr/rds_modernm.zip" ];
+	then
+		# Note that this is an older rds_modernm.zip that is approximate 2 GiB in size.
+		cp -f "${AUXILIARY_DATA_PATH}/nsrlsvr/rds_modernm.zip" .
+	fi
+
+	if [ ! -f rds_modernm.zip ];
+	then
+		# Download the minimum modern RDS hash set.
+		# Note that rds_modernm.zip is approximate 18 GiB in size.
+		curl -o rds_modernm.zip https://s3.amazonaws.com/rds.nsrl.nist.gov/RDS/rds_2024.03.1/RDS_2024.03.1_modern_minimal.zip
+	fi
+
+	if [ ! -f rds_modernm.zip ];
+	then
+		echo "Missing: rds_modernm.zip";
+
+		exit 1
+	fi
+
+	unzip -x rds_modernm.zip rds_modernm/NSRLFile.txt
+
+	mv rds_modernm/NSRLFile.txt .
+fi
+
+if [ ! -f NSRLFile.txt ];
+then
+	echo "Missing: NSRLFile.txt";
+
+	exit 1
+fi
+
+docker build -f nsrlsvr.Dockerfile --force-rm --no-cache -t log2timeline/nsrlsvr . ;
+
+# Update the nsrlsvr hashes.txt file from NSRLFile.txt
+docker run -v "${AUXILIARY_DATA_PATH}:/data:z" log2timeline/nsrlsvr /bin/bash -c "/usr/bin/python3 /usr/bin/nsrlupdate /data/NSRLFile.txt";
+
+# Preserver the intermediate container so we don't have to rebuild hashes.txt
+docker commit `docker ps -lq` | cut -c8- > nsrlsvr-container
+
+# Run nsrlsvr self tests
+docker run `cat nsrlsvr-container` /bin/bash -c "time /usr/bin/nsrlsvr --dry-run";
+
+# Run nsrlsvr listening on port 9120
+docker run -d --name=nsrlsvr --network=nsrlsvr-network -p 127.0.0.1:9120:9120 `cat nsrlsvr-container` /bin/bash -c "/usr/bin/nsrlsvr -p 9120";
+
+# Give nsrlsvr 1 minute to load the hashes and be ready for action
+# `time nsrlsvr --dry-run' indicates this takes about 30 seconds
+sleep 60
+
diff --git a/config/linux/ubuntu_install_opensearch.sh b/config/linux/ubuntu_install_opensearch.sh
deleted file mode 100755
index 8568e0c9c5..0000000000
--- a/config/linux/ubuntu_install_opensearch.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-#
-# Script to install OpenSearch on Ubuntu
-
-# Exit on error.
-set -e
-
-OPENSEARCH_VERSION="1.2.2";
-
-sudo apt-get update
-sudo apt-get install -y wget
-
-# TODO: update /etc/sysctl.conf
-# vm.max_map_count=262144
-
-adduser opensearch
-
-# Download OpenSearch
-
-wget -q https://artifacts.opensearch.org/releases/bundle/opensearch/${OPENSEARCH_VERSION}/opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz
-
-# Install OpenSearch
-
-tar xfv ${PWD}/opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz
-
-# Install the OpenSearch Python bindings
-
-sudo apt-get install -y python3-opensearch
-
-# Start OpenSearch
-
-cd opensearch-${OPENSEARCH_VERSION}
-
-echo "Starting OpenSearch";
-
-su -c './opensearch-tar-install.sh -Eplugins.security.disabled=true' opensearch &