Bringing in Willems changes to LCM

CHANGELOG.MD

@@ -0,0 +1,68 @@
+<H1>Changelog stageworkshop</H1>
+
+* Unreleased
+
+* Version 2.0.6-ci.14
+
+<H2>Changes made</H2>
+
+
+* 2019-04-04 [email protected]
+
+<H3>Stageworkshop part</H3>
+  * Changed the following files:
+
+    * scripts/lib.pc.sh; Added LCM upgrades using API calls and not files. Also added Karbon enable
+
+<H3>Call back server part</H3>
+
+  * No changes
+
+
+<H2>Changes made</H2>
+
+
+* 2019-03-21 [email protected]
+
+<H3>Stageworkshop part</H3>
+  * Changed the following files:
+
+    * scripts/lib.pc.sh; Added a loop function for the waiting for Calm to be enabled before starting the LCM.
+
+<H3>Call back server part</H3>
+
+  * No changes
+
+
+* 2019-03-20 [email protected]
+
+<H3>Stageworkshop part</H3>
+  * Changed the following files:
+
+    * release.json; Changed all the 13 to 14 as a number
+    * scripts/lib.pc.sh; Added a loop function for the LCM part to follow the progress of teh API call. Also added an extra wait of 5 minutes so that the "Enable Application" is done before we start the LCM! Otherwise it will crash.
+    * script/lib/pe/sh; Added a line that if it's being called by we- something, it changes lib.common.sh to we-lib.common.sh
+
+  * Copied for keeping the original the following files:
+
+    * scripts/lib.pc.org.sh
+
+  * Created extra files for a server reporting system (centralised logging using curl)
+
+    * we_stage_workshop.sh; the new version of the staging workshop script. This one holds the lines for the centralised feedback
+    * we_push_centos_cl_disk.sh; small script to push the CentOS Image that is needed in the TS2019 workshops
+    * script/we-ts2019.sh; has a more debug info fired for the scripts run at PE and PC timeframe (nohup bash -x)
+    * script/we-lib.common.sh; contains extra functions for the call back to the server 
+
+
+<H3>Call back server part</H3>
+  * Reason for this server is to have the possibility to send the logging/steps of the stageworkshop script to a centralised server. The "tool" is based on python.
+
+  * Created files and their goal:
+
+    * logserver/logserver.py; Listener for the messagees from the scripts.
+      * usage: python3 logserver.py 3000 to start the logserver on port 3000 (used by the scripts for now)
+    * logserver/webserver.py; Webserver to show the messages of the different running scripts
+      * usage: python3 webserver to start the webserver which will default to port 5000 by default.
+
+

cluster.txt

@@ -0,0 +1 @@
+10.42.86.37|techX2019!|[email protected]

logserver/hpoc_deploy.sql

@@ -0,0 +1,61 @@
+-- MySQL dump 10.14  Distrib 5.5.60-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: hpoc_deploy
+-- ------------------------------------------------------
+-- Server version	5.5.60-MariaDB
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Current Database: `hpoc_deploy`
+--
+
+CREATE DATABASE /*!32312 IF NOT EXISTS*/ `hpoc_deploy` /*!40100 DEFAULT CHARACTER SET latin1 */;
+
+USE `hpoc_deploy`;
+
+--
+-- Table structure for table `deploy_status`
+--
+
+DROP TABLE IF EXISTS `deploy_status`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `deploy_status` (
+  `id` int(6) NOT NULL AUTO_INCREMENT,
+  `hostname` varchar(255) DEFAULT NULL,
+  `module` varchar(255) DEFAULT NULL,
+  `replycode` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=129 DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `deploy_status`
+--
+
+LOCK TABLES `deploy_status` WRITE;
+/*!40000 ALTER TABLE `deploy_status` DISABLE KEYS */;
+INSERT INTO `deploy_status` VALUES (125,'10.42.100.37','dependencies','Warning: assuming on PC or PE VM, removing jq...'),(126,'10.42.21.37','dependencies','Warning: assuming on PC or PE VM, removing jq...'),(127,'10.42.2.37','dependencies','Warning: assuming on PC or PE VM, removing jq...'),(128,'10.42.31.37','dependencies','Warning: assuming on PC or PE VM, removing jq...');
+/*!40000 ALTER TABLE `deploy_status` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2019-03-13  3:49:55

logserver/logserver.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import logging
+import urllib
+import mysql.connector as mariadb
+from flask import Flask
+
+class S(BaseHTTPRequestHandler):
+    def _set_response(self):
+        self.send_response(200)
+        self.send_header('Content-type', 'text/html')
+        self.end_headers()
+
+    def do_POST(self):
+        content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
+        post_data = self.rfile.read(content_length) # <--- Gets the data itself
+        #logging.info("POST request,\nPath: %s\nHeaders:\n%s\n\nBody:\n%s\n",str(self.path), str(self.headers), post_data.decode('utf-8'))
+        message=urllib.parse.unquote(self.path)
+        message_list=message[1:].split("|")
+        db_actions(message_list[0],message_list[2],message_list[3],message_list[4],'insert')
+
+        self._set_response()
+        #self.wfile.write("POST request for {}".format(self.path).encode('utf-8'))
+
+# MariaDB related stuff. get the query and what is the module to run (create,insert, update, etc)
+def db_actions(date,host_ip,module,module_msg,action):
+    # open the mariadb connection
+    mariadb_connection=mariadb.connect(user='webusr',password='webusr',database='hpoc_deploy',host='127.0.0.1',port='3306')
+    cursor=mariadb_connection.cursor()
+
+    # check if the host_ip exists in the database
+    query='select count(*) from deploy_status where hostname=\'' + host_ip + '\''
+    cursor.execute(query)
+
+    row=cursor.fetchone()
+    if row[0]==0:
+        # if the hostname does not exist in the table, add it to the table and move forward
+        query="insert into deploy_status(hostname) values(\'" + host_ip +"\')"
+        cursor.execute(query)
+        mariadb_connection.commit()
+    else:
+        #update deploy_status set replycode='Cluster status: ', module='cluster_check' where hostname='10.42.100.37'
+        query="update deploy_status set replycode=\'" + module_msg +"\', module=\'" + module + "\' where hostname=\'" + host_ip +"\'" 
+        cursor.execute(query)
+        mariadb_connection.commit()
+
+    #close the mariadb connection
+    mariadb_connection.close()
+    return
+
+# Function for running the HTTP server
+def run(server_class=HTTPServer, handler_class=S, port=8080):
+    logging.basicConfig(level=logging.INFO)
+    server_address = ('', port)
+    httpd = server_class(server_address, handler_class)
+    logging.info('Starting httpd...\n')
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    httpd.server_close()
+    logging.info('Stopping httpd...\n')
+
+if __name__ == '__main__':
+    from sys import argv
+
+    if len(argv) == 2:
+        run(port=int(argv[1]))
+    else:
+        run()

logserver/nagios_setup.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Script for setting up the configuration files for the PE Cluster during the TechSummit 2019
+
+
+# Get the list of the clusetrs to be created
+for ip_name in `cat /root/scripts/pelist.txt`
+do
+	# Grab the IP address from the cluster
+	pe_ip=`echo $ip_name | cut -d";" -f 2`
+
+	# Grab the name of the cluster
+	pe_name=`echo $ip_name | cut -d";" -f 1`
+
+	# Copy the default file to the new, good hostname
+	cp /usr/local/nagios/etc/servers/yourhost.cfg /usr/local/nagios/etc/servers/$pe_name.cfg 
+
+	# Changing the original files to the right information
+	# Set the right Servername
+	sed -i "s/CLUSTER_NAME/$pe_name/g" /usr/local/nagios/etc/servers/$pe_name.cfg
+
+	# Set the right IP address
+	sed -i "s/CLUSTER_IP/$pe_ip/g" /usr/local/nagios/etc/servers/$pe_name.cfg
+done
+
+# Rename the temp file
+mv /usr/local/nagios/etc/servers/yourhost.cfg /usr/local/nagios/etc/servers/yourhost.cfg.tmp
+
+# Restart Nagios so it can start monitoring:
+systemctl reload nagios.service

logserver/pelist.txt

@@ -0,0 +1,84 @@
+PHX-POC001;10.42.1.37
+PHX-POC002;10.42.2.37
+PHX-POC003;10.42.3.37
+PHX-POC004;10.42.4.37
+PHX-POC005;10.42.5.37
+PHX-POC006;10.42.6.37
+PHX-POC007;10.42.7.37
+PHX-POC010;10.42.10.37
+PHX-POC011;10.42.11.37
+PHX-POC012;10.42.12.37
+PHX-POC013;10.42.13.37
+PHX-POC014;10.42.14.37
+PHX-POC015;10.42.15.37
+PHX-POC016;10.42.16.37
+PHX-POC019;10.42.19.37
+PHX-POC020;10.42.20.37
+PHX-POC021;10.42.21.37
+PHX-POC022;10.42.22.37
+PHX-POC024;10.42.24.37
+PHX-POC025;10.42.25.37
+PHX-POC026;10.42.26.37
+PHX-POC027;10.42.27.37
+PHX-POC028;10.42.28.37
+PHX-POC029;10.42.29.37
+PHX-POC030;10.42.30.37
+PHX-POC031;10.42.31.37
+PHX-POC032;10.42.32.37
+PHX-POC033;10.42.33.37
+PHX-POC034;10.42.34.37
+PHX-POC037;10.42.37.37
+PHX-POC038;10.42.38.37
+PHX-POC042;10.42.42.37
+PHX-POC044;10.42.44.37
+PHX-POC045;10.42.45.37
+PHX-POC046;10.42.46.37
+PHX-POC047;10.42.47.37
+PHX-POC048;10.42.48.37
+PHX-POC049;10.42.49.37
+PHX-POC050;10.42.50.37
+PHX-POC051;10.42.51.37
+PHX-POC052;10.42.52.37
+PHX-POC053;10.42.53.37
+PHX-POC054;10.42.54.37
+PHX-POC055;10.42.55.37
+PHX-POC056;10.42.56.37
+PHX-POC057;10.42.57.37
+PHX-POC061;10.42.61.37
+PHX-POC062;10.42.62.37
+PHX-POC065;10.42.65.37
+PHX-POC066;10.42.66.37
+PHX-POC067;10.42.67.37
+PHX-POC068;10.42.68.37
+PHX-POC069;10.42.69.37
+PHX-POC070;10.42.70.37
+PHX-POC072;10.42.72.37
+PHX-POC073;10.42.73.37
+PHX-POC074;10.42.74.37
+PHX-POC079;10.42.79.37
+PHX-POC080;10.42.80.37
+PHX-POC081;10.42.81.37
+PHX-POC082;10.42.82.37
+PHX-POC083;10.42.83.37
+PHX-POC086;10.42.86.37
+PHX-POC087;10.42.87.37
+PHX-POC088;10.42.88.37
+PHX-POC090;10.42.90.37
+PHX-POC091;10.42.91.37
+PHX-POC092;10.42.92.37
+PHX-POC093;10.42.93.37
+PHX-POC094;10.42.94.37
+PHX-POC095;10.42.95.37
+PHX-POC096;10.42.96.37
+PHX-POC097;10.42.97.37
+PHX-POC098;10.42.98.37
+PHX-POC099;10.42.99.37
+PHX-POC101;10.42.101.37
+PHX-POC102;10.42.102.37
+PHX-POC104;10.42.104.37
+PHX-POC106;10.42.106.37
+PHX-POC107;10.42.107.37
+PHX-POC108;10.42.108.37
+PHX-POC109;10.42.109.37
+PHX-POC110;10.42.110.37
+PHX-POC113;10.42.113.37

logserver/templates/index.html

@@ -0,0 +1,50 @@
+<html>
+    <head><title>Stageworkshop Summary</title>
+        <meta http-equiv="refresh" content="5" >
+    </head>
+<body>
+<style>
+    table {
+        font-family: arial, sans-serif;
+        border-collapse: collapse;
+        width: 100%;
+    }
+    td, th {
+        border: 1px solid #dddddd;
+        text-align: left;
+        padding: 8px;
+    }
+    tr:nth-child(even) {
+        background-color: #dddddd;
+    }
+</style>
+<meta charset="UTF-8">
+<title>Employees</title>
+<table id="messages" style="width:100%">
+        <tbody>
+            <tr>
+                <th>Hostname</th>
+                <th>Module</th>
+                <th>Log message</th>
+            </tr>
+
+{% if result %}
+
+{% for row in result %}
+
+    <tr>
+        <td> {{ row[0] }}</td>
+        <td> {{ row[1] }}</td>
+        <td> {{ row[2] }}</td>
+    </tr>
+
+{% endfor %}
+
+{% endif %}
+
+
+
+    </tbody>
+</table>
+</body>
+</html>