Changes for end-to-end tests

config/end_to_end/nsrlsvr.Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:noble
+FROM ubuntu:jammy
 MAINTAINER Log2Timeline <log2timeline-dev@go
 
 ENV DEBIAN_FRONTEND=noninteractive

config/end_to_end/run_tests_with_docker.sh

@@ -73,25 +73,10 @@ do
 
 	# TODO: move custom test setup and teardown scripts to configuration parameter?
 
-	if [[ ${OUTPUT_FORMAT} == "opensearch" ]] || [[ ${OUTPUT_FORMAT} == "opensearch_ts" ]];
-	then
-		# Install OpenSearch and give it 3 minutes to start-up before running the output end-to-end test.
-		COMMAND="./config/linux/ubuntu_install_opensearch.sh && sleep 3m && ${COMMAND}";
-	fi
-
 	if [[ ${TEST_NAME} == "acserver-mounted" ]];
 	then
 		COMMAND="mkdir -p /mnt/acserver_mount && mount -o ro,noload,noacl,loop,offset=1048576 /sources/acserver.dd /mnt/acserver_mount && ./tests/end-to-end.py --config /config/${TEST_NAME}.ini --references-directory test_data/end_to_end --results-directory plaso-out --sources-directory /mnt --scripts-directory plaso/scripts && umount /mnt/acserver_mount && rmdir /mnt/acserver_mount";
 
-	elif [[ ${TEST_NAME} == *\-nsrlsvr ]];
-	then
-		# Install nsrlsvr and give it 3 minutes to start-up before running the output end-to-end test.
-		COMMAND="./config/linux/ubuntu_install_nsrlsvr.sh && sleep 3m && ${COMMAND}";
-
-	elif [[ ${TEST_NAME} == *\-redis ]];
-	then
-		# TODO: add support for Redis tests
-		continue;
 	fi
 	echo "Running ${TEST_CASE} end-to-end test: ${TEST_NAME}";
 

config/jenkins/run_end_to_end_tests.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Script to run end-to-end tests on a Linux Jenkins instance with Docker.
+# Script to run end-to-end tests on an Ubuntu Jenkins instance with Docker.
 
 # Fail on error.
 set -e
@@ -33,7 +33,11 @@ docker run log2timeline/plaso ./utils/check_dependencies.py;
 
 COMMAND="./tests/end-to-end.py --config /config/${CONFIGURATION_NAME}.ini --references-directory test_data/end_to_end --results-directory /home/test/plaso/plaso-out --sources-directory /sources --scripts-directory plaso/scripts";
 
-if test ${CONFIGURATION_NAME} = "output_opensearch";
+if test ${CONFIGURATION_NAME} = "psort-studentpc1-nsrlsvr";
+then
+	DOCKER_NETWORK="--network=nsrlsvr-network";
+
+elif test ${CONFIGURATION_NAME} = "output_opensearch" || test ${CONFIGURATION_NAME} = "output_opensearch_ts";
 then
 	DOCKER_NETWORK="--network=opensearch-network";
 

config/jenkins/run_nsrlsvr.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+#
+# Script to run nsrlsvr on an Ubuntu Jenkins instance with Docker.
+
+AUXILIARY_DATA_PATH="/media/auxiliary";
+
+# Exit on error.
+set -e
+
+sudo apt-get install -y curl unzip
+
+if [ -f "${AUXILIARY_DATA_PATH}/nsrlsvr/NSRLFile.txt" ];
+then
+	# Note that NSRLFile.txt is approximate 4 GiB in size.
+	cp -f "${AUXILIARY_DATA_PATH}/nsrlsvr/NSRLFile.txt" .
+fi
+
+if [ ! -f NSRLFile.txt ];
+then
+	if [ -f "${AUXILIARY_DATA_PATH}/nsrlsvr/rds_modernm.zip" ];
+	then
+		# Note that this is an older rds_modernm.zip that is approximate 2 GiB in size.
+		cp -f "${AUXILIARY_DATA_PATH}/nsrlsvr/rds_modernm.zip" .
+	fi
+
+	if [ ! -f rds_modernm.zip ];
+	then
+		# Download the minimum modern RDS hash set.
+		# Note that rds_modernm.zip is approximate 18 GiB in size.
+		curl -o rds_modernm.zip https://s3.amazonaws.com/rds.nsrl.nist.gov/RDS/rds_2024.03.1/RDS_2024.03.1_modern_minimal.zip
+	fi
+
+	if [ ! -f rds_modernm.zip ];
+	then
+		echo "Missing: rds_modernm.zip";
+
+		exit 1
+	fi
+
+	unzip -x rds_modernm.zip rds_modernm/NSRLFile.txt
+
+	mv rds_modernm/NSRLFile.txt .
+fi
+
+if [ ! -f NSRLFile.txt ];
+then
+	echo "Missing: NSRLFile.txt";
+
+	exit 1
+fi
+
+docker build -f nsrlsvr.Dockerfile --force-rm --no-cache -t log2timeline/nsrlsvr . ;
+
+# Update the nsrlsvr hashes.txt file from NSRLFile.txt
+docker run -v "${AUXILIARY_DATA_PATH}:/data:z" log2timeline/nsrlsvr /bin/bash -c "/usr/bin/python3 /usr/bin/nsrlupdate /data/NSRLFile.txt";
+
+# Preserver the intermediate container so we don't have to rebuild hashes.txt
+docker commit `docker ps -lq` | cut -c8- > nsrlsvr-container
+
+# Run nsrlsvr self tests
+docker run `cat nsrlsvr-container` /bin/bash -c "time /usr/bin/nsrlsvr --dry-run";
+
+# Run nsrlsvr listening on port 9120
+docker run -d --name=nsrlsvr --network=nsrlsvr-network -p 127.0.0.1:9120:9120 `cat nsrlsvr-container` /bin/bash -c "/usr/bin/nsrlsvr -p 9120";
+
+# Give nsrlsvr 1 minute to load the hashes and be ready for action
+# `time nsrlsvr --dry-run' indicates this takes about 30 seconds
+sleep 60
+