Update ELK to fix broken input behaviour

PNDA-4238

pillar/services.sls

@@ -1,12 +1,12 @@
 logstash:
-  version: 5.2.2
+  version: 6.2.1
 
 kibana:
-  version: 4.1.6-linux-x64
+  version: 6.2.1-linux-x86_64
   directory: /opt/pnda
 
 elasticsearch:
-  version: 1.5.0
+  version: 6.2.1
   directory: /opt/pnda
   logdir: /var/log/elasticsearch
   confdir: /etc/elasticsearch

salt/curator/files/action.yml

@@ -0,0 +1,20 @@
+actions:
+  1:
+    action: delete_indices
+    description: >-
+      Delete indices older than 1 day (based on index name), for logstash-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: logstash-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: 1

salt/curator/files/config.yml

@@ -0,0 +1,19 @@
+client:
+  hosts:
+    - 127.0.0.1
+  port: 9200
+  url_prefix:
+  use_ssl: False
+  certificate:
+  client_cert:
+  client_key:
+  ssl_no_validate: False
+  http_auth:
+  timeout: 30
+  master_only: False
+
+logging:
+  loglevel: INFO
+  logfile: /var/log/pnda/curator/curator.log
+  logformat: default
+  blacklist: ['elasticsearch', 'urllib3']

salt/curator/files/requirements.txt

@@ -1,4 +1,4 @@
-click==6.6
-elasticsearch==2.4.0
-elasticsearch-curator==3.5.1
+click==6.7
+elasticsearch==6.1.1
+elasticsearch-curator==5.4.1
 urllib3==1.19.1

salt/curator/init.sls

@@ -14,10 +14,28 @@ curator-python-elasticsearch-curator:
     - require:
       - pip: python-pip-install_python_pip
 
+curator-logs-dir:
+  file.directory:
+    - name: /var/log/pnda/curator
+    - mode: 755
+    - makedirs: True
+    - recurse:
+      - mode
+
+curator-deploy-config:
+  file.managed:
+    - name: {{ virtual_env_dir }}/config.yml
+    - source: salt://curator/files/config.yml
+
+curator-deploy-action:
+  file.managed:
+    - name: {{ virtual_env_dir }}/action.yml
+    - source: salt://curator/files/action.yml
+
 curator-update-crontab-inc-curator:
   cron.present:
     - identifier: CURATOR-DELETE-INDICES
     - user: root
     - minute: 01
     - hour: 00
-    - name: {{ virtual_env_dir }}/bin/curator delete indices --older-than {{ flavor_cfg.days_to_keep }} --time-unit days --prefix logstash- --timestring \%Y.\%m.\%d >> /tmp/curator.log 2>&1
+    - name: {{ virtual_env_dir }}/bin/curator --config {{ virtual_env_dir }}/config.yml {{ virtual_env_dir }}/action.yml

salt/data-service/files/requirements.txt

@@ -1,6 +1,6 @@
 backports-abc==0.5
 backports.ssl-match-hostname==3.5.0.1
-certifi==2016.9.26
+certifi==2017.4.17
 cm-api==14.0.0
 functools32==3.2.3.post2
 futures==3.0.5

salt/elasticsearch/init.sls

@@ -63,7 +63,7 @@ elasticsearch-dl_and_extract_elasticsearch:
   archive.extracted:
     - name: {{ es_p.directory }}
     - source: {{ elasticsearch_url }}
-    - source_hash: {{ elasticsearch_url }}.sha1.txt
+    - source_hash: {{ elasticsearch_url }}.sha512
     - user: elasticsearch
     - group: elasticsearch
     - archive_format: tar

salt/elasticsearch/templates/elasticsearch.init.conf.tpl

@@ -23,4 +23,4 @@ end script
 
 chdir {{ installdir }}
 
-exec bin/elasticsearch -Des.default.config={{ defaultconfig }} -Des.default.path.logs={{ logdir }} -Des.default.path.data={{ datadir }} -Des.default.path.work={{ workdir }} -Des.default.path.conf={{ confdir }}
+exec bin/elasticsearch -Epath.logs={{ logdir }} -Epath.data={{ datadir }}

salt/elasticsearch/templates/elasticsearch.service.tpl

@@ -4,7 +4,8 @@ Description=elasticsearch service
 [Service]
 Type=simple
 LimitNOFILE=32768
-ExecStart={{ installdir }}/bin/elasticsearch -Des.default.config={{ defaultconfig }} -Des.default.path.logs={{ logdir }} -Des.default.path.data={{ datadir }} -Des.default.path.work={{ workdir }} -Des.default.path.conf={{ confdir }}
+User=elasticsearch
+ExecStart={{ installdir }}/bin/elasticsearch -Epath.logs={{ logdir }} -Epath.data={{ datadir }}
 Restart=always
 RestartSec=5
 

salt/kibana/files/import-dashboard.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+k_result=1
+num_tries=30
+while [ $k_result -ne 0 ] && [ $num_tries -gt 0 ]
+do
+  echo "Checking kibana connectivity ($num_tries remaining)..."
+  curl -f 'localhost:5601'
+  k_result=$?
+  if [ $k_result -ne 0 ]; then
+    sleep 5
+    num_tries=$(($num_tries-1))
+  fi
+done
+
+curl -X POST -H "Content-Type: application/json" -H "kbn-xsrf: true" -d @dashboard.json http://localhost:5601/api/kibana/dashboards/import

salt/kibana/files/kibana.yml

@@ -1,20 +1,20 @@
 # Kibana is served by a back end server. This controls which port to use.
-port: 5601
+server.port: 5601
 
 # The host to bind the server to.
-host: "0.0.0.0"
+server.host: "0.0.0.0"
 
 # The Elasticsearch instance to use for all your queries.
 #elasticsearch_url: "http://localhost:9200"
-elasticsearch_url: "http://localhost:9200"
+elasticsearch.url: "http://localhost:9200"
 
 # preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false,
 # then the host you use to connect to *this* Kibana instance will be sent.
-elasticsearch_preserve_host: true
+elasticsearch.preserveHost: true
 
 # Kibana uses an index in Elasticsearch to store saved searches, visualizations
 # and dashboards. It will create a new index if it doesn't already exist.
-kibana_index: ".kibana"
+kibana.index: ".kibana"
 
 # If your Elasticsearch is protected with basic auth, this is the user credentials
 # used by the Kibana server to perform maintence on the kibana_index at statup. Your Kibana
@@ -32,36 +32,23 @@ kibana_index: ".kibana"
 # ca: /path/to/your/CA.pem
 
 # The default application to load.
-default_app_id: "dashboard/Default"
+kibana.defaultAppId: "dashboard/a2521e70-1622-11e8-9c84-9713efe53d19"
 
 # Time in milliseconds to wait for responses from the back end or elasticsearch.
 # This must be > 0
-request_timeout: 300000
+elasticsearch.requestTimeout: 300000
 
 # Time in milliseconds for Elasticsearch to wait for responses from shards.
 # Set to 0 to disable.
-shard_timeout: 0
+elasticsearch.shardTimeout: 0
 
 # Set to false to have a complete disregard for the validity of the SSL
 # certificate.
-verify_ssl: true
+elasticsearch.ssl.verificationMode: certificate
 
 # SSL for outgoing requests from the Kibana Server (PEM formatted)
 # ssl_key_file: /path/to/your/server.key
 # ssl_cert_file: /path/to/your/server.crt
 
 # Set the path to where you would like the process id file to be created.
 # pid_file: /var/run/kibana.pid
-
-# Plugins that are included in the build, and no longer found in the plugins/ folder
-bundled_plugin_ids:
- - plugins/dashboard/index
- - plugins/discover/index
- - plugins/doc/index
- - plugins/kibana/index
- - plugins/markdown_vis/index
- - plugins/metric_vis/index
- - plugins/settings/index
- - plugins/table_vis/index
- - plugins/vis_types/index
- - plugins/visualize/index

salt/kibana/init.sls

@@ -21,7 +21,7 @@ kibana-dl_and_extract_kibana:
   archive.extracted:
     - name: {{ kibana_directory }}
     - source: {{ kibana_url }}
-    - source_hash: {{ kibana_url }}.sha1.txt
+    - source_hash: {{ kibana_url }}.sha512
     - user: kibana
     - group: kibana
     - archive_format: tar

salt/kibana/kibana-dashboard.sls

@@ -1,7 +1,18 @@
+{% set kibana_version = pillar['kibana']['version'] %}
+{% set kibana_directory = salt['pillar.get']('kibana:directory', '/opt/pnda') + '/kibana-' + kibana_version %}
+
 kibana-dashboard-running:
   service.running:
     - name: kibana
 
-kibana-dashboard-configure:
+kibana-dashboard_stage:
+  file.managed:
+    - name: {{ kibana_directory }}/dashboard.json
+    - source: salt://kibana/files/dashboard.json
+    - user: kibana
+    - group: kibana
+
+kibana-dashboard-import:
   cmd.script:
-    - source: salt://kibana/files/configure-dashboard.sh
+    - source: salt://kibana/files/import-dashboard.sh
+    - cwd: {{ kibana_directory }}

salt/logserver/logserver_templates/logstash.conf.tpl

@@ -5,4 +5,4 @@ respawn limit unlimited
 post-stop exec sleep 2
 env confFile={{ install_dir }}/logstash/collector.conf
 env programDir={{ install_dir }}/logstash
-exec ${programDir}/bin/logstash -f ${confFile}
+exec ${programDir}/bin/logstash --path.data {{ install_dir }}/logstash/logserver-data -f ${confFile}

salt/logserver/logserver_templates/logstash.service.tpl

@@ -3,7 +3,7 @@ Description=logserver service
 
 [Service]
 Type=simple
-ExecStart={{ install_dir }}/logstash/bin/logstash -f {{ install_dir }}/logstash/collector.conf
+ExecStart={{ install_dir }}/logstash/bin/logstash --path.data {{ install_dir }}/logstash/logserver-data -f {{ install_dir }}/logstash/collector.conf
 Restart=always
 RestartSec=2
 

salt/logserver/logshipper.sls

@@ -7,7 +7,7 @@
 {% set logstash_version = pillar['logstash']['version'] %}
 {% set logstash_package = 'logstash-' + logstash_version + '.tar.gz' %}
 {% set logstash_url = mirror_location + logstash_package %}
-{% set plugin_pack_name = 'logstash-offline-plugins-5.2.2.zip' %}
+{% set plugin_pack_name = 'logstash-offline-plugins-6.2.1.zip' %}
 {% set plugin_pack_url = mirror_location + plugin_pack_name %}
 
 include:
@@ -29,7 +29,7 @@ logshipper-dl-and-extract:
   archive.extracted:
     - name: {{ install_dir }}
     - source: {{ logstash_url }}
-    - source_hash: {{ logstash_url }}.sha1
+    - source_hash: {{ logstash_url }}.sha512
     - archive_format: tar
     - tar_options: ''
     - if_missing: {{ install_dir }}/logstash-{{ logstash_version }}

salt/logserver/logshipper_templates/logstash.conf.tpl

@@ -3,7 +3,6 @@ stop on runlevel [016]
 respawn
 respawn limit unlimited
 post-stop exec sleep 2
-setuid logger
 env confFile={{ install_dir }}/logstash/shipper.conf
 env programDir={{ install_dir }}/logstash
-exec ${programDir}/bin/logstash -f ${confFile}
+exec ${programDir}/bin/logstash --path.data {{ install_dir }}/logstash/logshipper-data -f ${confFile}

salt/logserver/logshipper_templates/logstash.service.tpl

@@ -3,7 +3,7 @@ Description=Logstash
 
 [Service]
 Type=simple
-ExecStart={{ install_dir }}/logstash/bin/logstash -f {{ install_dir }}/logstash/shipper.conf
+ExecStart={{ install_dir }}/logstash/bin/logstash --path.data {{ install_dir }}/logstash/logshipper-data -f {{ install_dir }}/logstash/shipper.conf
 Restart=always
 RestartSec=2
 

salt/logserver/logshipper_templates/shipper.conf.tpl

@@ -148,6 +148,8 @@ input {
 
 filter {
    if [_systemd_unit] {
+       mutate { remove_field => [ "_uid"] }
+
        if [_systemd_unit] == "kafka.service" {
            mutate {add_field => {"source" => "kafka"}}
        }

salt/logstash/init.sls

@@ -63,7 +63,7 @@ logstash-dl_and_extract_logstash:
   archive.extracted:
     - name: {{logstash_directory}}
     - source: {{ logstash_url }}
-    - source_hash: {{ logstash_url }}.sha1
+    - source_hash: {{ logstash_url }}.sha512
     - archive_format: tar
     - tar_options: ''
     - if_missing: {{logstash_directory}}/logstash-{{ logstash_version }}