Added elk template, updates to use loganalyzer with logserver template

Makefile.global

@@ -9,7 +9,7 @@ SSH_KEY +="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDqvpOmtZTt5vGjKEbE+xF9RpjnhHa6c
 SSH_KEY +="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC7CAJhyapph+OQV3evDflSdHn9zRY+caRj3EqhcTgLQItZgDjYUxwW4zEWKAFuASG0E8Df3caPPkLGOZx0qhy2lBQAufOdejboMwMSoSahP2Mlz8+0zIGS8T/4J5mDhm/hX6l1pMMWZM69iLENwOjrOFI64453CYVBiDF3trEuP9ybc9Yj595+YPcUin8Ak/cYsAkRjP2V5LG3MqisGFyKx5I0IAGFOsRda5r9ogeWGKLbO4qAHx/L2RxiCX9gCiFquaqzwCbE3XEOe0tY4eD6K6CpfgAd1MKivDm3yqGKcUZ5vq8xrnnIENx9tE4VloPv0I/y40LzFW03rGUsXlYUrrQyzwj3OautM6pEV5k5bejs9Qe9Znxb15G5+ffRcAowf9TeN5JnZ+pP6htgWqjuIr21gibvzAQweFANV5l8LpnrH/ezwNRgN1gDGDPTJEDa7i++wNIyOVacJoGwMueOlsqc+RQfH7zHXvqz1gRgvUf31gSboFRhmwTawlPCRW4/MAD7A8TlNnpRqAn1hK1/dAn+WSDPvAu9ZxiU0Z2A3ixmnMEOLLcMil/5JhkYcCSFcfO445p455oCB5d473pejm4WZkM0Xg277ArrC2eqVppTCftWjiZNNEE6zEoQwWaWpjZolvX56omG34QSMRx+SmrztZv4rzBJFnwNWv1WRQ== [email protected]\n"
 
 # Add standard packages that you would like by default on all templates
-PACKAGES += sudo tcpdump traceroute
+PACKAGES += chrony sudo tcpdump traceroute
 
 # Set this to 1 to permit root login over SSH.
 # This is important if you are using Proxmox without a public SSH key, as 

elk/.gitignore

@@ -0,0 +1,6 @@
+.veid
+config
+info
+logfile
+rootfs
+ubuntu-16.04-elk_16.04-1_amd64.tar.gz

elk/dab.conf

@@ -0,0 +1,15 @@
+Suite: xenial
+CacheDir: ../cache
+Source: http://archive.ubuntu.com/ubuntu SUITE main restricted universe multiverse 
+Source: http://archive.ubuntu.com/ubuntu SUITE-updates main restricted universe multiverse  
+Source: http://archive.ubuntu.com/ubuntu SUITE-security main restricted universe multiverse 
+Source: http://artifacts.elastic.co/packages/5.x/apt stable main
+Source: http://ppa.launchpad.net/webupd8team/java/ubuntu SUITE main
+Architecture: amd64
+Name: elk
+Version: 16.04-1
+Section: system
+Maintainer: Nathan Gardiner <[email protected]>
+Infopage: http://www.elastic.co
+Description: Elasticsearch, Logstash, Kibana (ELK) stack
+ Provides advanced analysis and search features for log data.

logserver/.gitignore

@@ -4,4 +4,5 @@ info
 logfile
 rootfs
 jdk-8u121-linux-x64.tar.gz
-ubuntu-16.04-ansible_16.04-1_amd64.tar.gz
+loganalyzer-4.1.5.tar.gz
+ubuntu-16.04-logserver_16.04-1_amd64.tar.gz

logserver/Makefile

@@ -16,8 +16,12 @@ bootstrap:
 	# Install Apache now, because we'll use it to avoid having to have
 	# internet access for the Java installation
 	dab install apache2
-	dab exec mkdir -p /var/www/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441
-	install -m 0700 jdk-8u121-linux-x64.tar.gz ${BASEDIR}/var/www/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-x64.tar.gz
+	install -m 0666 jdk-8u121-linux-x64.tar.gz ${BASEDIR}/var/www/html/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-x64.tar.gz
+
+	# Install loganalyzer under apache root
+	install -m 0666 loganalyzer-4.1.5.tar.gz ${BASEDIR}/var/www/html/loganalyzer-4.1.5.tar.gz
+	dab exec cd /var/www/html && tar xzf loganalyzer-4.1.5.tar.gz
+	dab exec mv /var/www/html/loganalyzer-4.1.5 /var/www/html/loganalyzer
 
 	# Install Oracle 8 and set JAVA_HOME which is a pre-requisite for
 	# installation of logstash
@@ -26,6 +30,10 @@ bootstrap:
 
 	dab install logstash rsyslog rsyslog-elasticsearch rsyslog-gnutls
 	dab install rsyslog-mysql rsyslog-relp
+
+	# Clean up after installation
+	dab exec rm -rf /var/www/html/otn-pub
+
 	install -m 0700 00-server.conf ${BASEDIR}/etc/rsyslog.d/00-server.conf
 	install -m 0700 runonce.sh ${BASEDIR}/etc/init.d/firstboot
 	dab exec update-rc.d firstboot defaults
@@ -41,6 +49,10 @@ ifeq ("$(wildcard jdk-8u121-linux-x64.tar.gz)","")
 	# Download the JDK binary in preparation for use during install
 	wget -c -O "jdk-8u121-linux-x64.tar.gz" --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-x64.tar.gz"
 endif
+ifeq ("$(wildcard loganalyzer-4.1.5.tar.gz)","")
+	# Download LogAnalyzer
+	wget -c -O "loganalyzer-4.1.5.tar.gz" --no-check-certificate http://download.adiscon.com/loganalyzer/loganalyzer-4.1.5.tar.gz
+endif
 
 info/init_ok: dab.conf
 	dab init

logserver/README.md

@@ -7,8 +7,11 @@
    - TCP 514 (TCP Syslog)
    - TCP 20514 (RELP - Reliable Log Protocol)
 
+- Includes a web interface (loganalyzer) for viewing logs from your web browser.
+
 - Includes Java 8 + Logstash for integration with ELK stack
    - Note: Logstash does not need to be used to integrate with Elasticsearch. Check the integration section below for more information
+   - See the elk template included in this repository for a template providing elasticsearch and kibana to be used alongside this template for full ELK functionality.
 
 - Adds any customizations such as root login enabled or SSH keys from ../Makefile.global
 - Total uncompressed image size is *588 MB*
@@ -24,8 +27,8 @@ To create the template archive:
 
 By default, the template will build as a standalone Log Server with listeners for standard UDP/TCP syslog and RELP reception. To enable other functionality, you can use the following options:
 
-- LSIP: Configures
-- RSIP: Configures
+- LSIP: Configures a Logstash connection to elasticsearch
+- RSIP: Configures a native rsyslog connection to elasticsearch
 
 To copy the template to the Proxmox VE Templates Directory:
 
@@ -39,7 +42,14 @@ To clean and return the template directory to original state
 
 ### Integration with devices/clients
 
-Nothing additional is required to integrate with your devices. Just point them to UDP/514 or TCP/514 for classic Syslog clients, or use RELP on port 20514 for a RELP-capable client
+Nothing additional is required to integrate with your devices. Just point them to UDP/514 or TCP/514 for classic Syslog clients, or use RELP on port 20514 for a RELP-capable client.
+
+### Integration with template containers
+
+The Makefile.global in the root directory of this repository provides a variable RELP_TARGETS which allows one or more RELP capable log servers such as this log server template to send all logs to. Configuring this variable will ensure that all templates created will begin logging to this logserver template out of the box.
 
 ### Integration with Elasticsearch/ELK
 
+Elasticsearch, logstash and kibana provide an advanced platform for searching, analysis and visualization of log data.
+
+This logserver template is designed to be able to be connected into an elasticsearch infrastructure for streaming of log data. To do this, you may use the make options listed under the Usage section, or to deploy the configuration you would like via an orchestration platform such as Ansible.

logserver/oracle-prereq.sh

@@ -6,16 +6,24 @@
 # Pause for a few seconds to ensure that there's no conflict with the dpkg
 # database, as background apt commands will lock it out and cause the script
 # to fail
-sleep 5
+sleep 20
 
 # Pre-seed the configuration for the license acceptance to ensure that the
 # installation does not pause
-debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections
-debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections
+echo "debconf shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
+echo "debconf shared/accepted-oracle-license-v1-1 seen true" | debconf-set-selections
 echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
 echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 seen true" | debconf-set-selections
 
 # We need to do a bit of trickery to download the package locally, as the
 # LXC template container will not have internet access during build.
 echo "127.0.0.1 download.oracle.com" >> /etc/hosts
+
+# Stop the range header from working, as this causes issues
+mkdir -p /var/www/html/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441
+echo "Header set Accept-Ranges none" > /var/www/html/otn-pub/java/jdk/8u121-b13/.htaccess
+echo "RequestHeader unset Range" >> /var/www/html/otn-pub/java/jdk/8u121-b13/.htaccess
+chmod 655 /var/www/html/otn-pub/java/jdk/8u121-b13/.htaccess
+
+# Set global pointer to the correct version of Java JRE
 echo "export JAVA_HOME=/usr/lib/jvm/java-8-oracle" >> /etc/environment