PNDA-2375 Isolate PNDA from breaking dependency changes

pndaproject · Apr 25, 2017 · f6a4f6c · f6a4f6c
1 parent 9954d69
commit f6a4f6c
Show file tree

Hide file tree

Showing 30 changed files with 267 additions and 131 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 - PNDA-2456: Initial work to support for Redhat 7. Salt highstate and orchestrate run on a Redhat7 HEAT cluster with no errors but requires further testing and work.
 - PNDA-2480: Added a per flavor pillar setting for kafka log retention (log.retention.bytes) set to 300MB (pico) 1GB (standard) to stop disks filling up on pico clusters.
 - PNDA-2682: review console backend deployment
+- PNDA-2375: Isolate PNDA from breaking dependency changes
 
 ### Changed
 - PNDA-2517: If Cloudera setup (cm_setup.py) fails, orchestrate can be re-run and cm_setup.py will attempt to continue from where it completed up to last time. Progress is recorded in /root/.CM_SETUP_SUCCESS which can be edited if manual control is required over the point to continue from.

diff --git a/pillar/packages/RedHat.sls b/pillar/packages/RedHat.sls
@@ -0,0 +1,89 @@
+acl:
+  package-name: acl
+  version: "2.2.51-12.el7"
+at:
+  package-name: at
+  version: "3.1.13-22.el7"
+cloudera-manager-agent:
+  package-name: cloudera-manager-agent
+  version: "5.10.0-1.cm5100.p0.85.el7"
+cloudera-manager-daemons:
+  package-name: cloudera-manager-daemons
+  version: "5.10.0-1.cm5100.p0.85.el7"
+cloudera-manager-server:
+  package-name: cloudera-manager-server
+  version: "5.10.0-1.cm5100.p0.85.el7"
+cyrus-sasl-devel:
+  package-name: cyrus-sasl-devel
+  version: "2.1.26-20.el7_2"
+g++:
+  package-name: gcc-c++
+  version: "4.8.5-11.el7"
+glibc-devel:
+  package-name: glibc-devel
+  version: "2.17-157.el7_3.1"
+gnuplot:
+  package-name: gnuplot
+  version: "4.6.2-3.el7"
+grafana:
+  package-source: 'grafana-3.1.1-1470047149.x86_64.rpm'
+libssl-dev:
+  package-name: openssl-devel
+  version: "1.0.1e-60.el7_3.1"
+libffi-dev:
+  package-name: libffi-devel
+  version:  "3.0.13-18.el7"
+libsasl:
+  package-name: libgsasl-devel
+  version: "1.8.0-8.el7"
+mysql-server:
+  package-name: mysql-community-server
+  version: "5.5.54-2.el7"
+  configuration_file: /etc/my.cnf
+  service_name: mysqld
+nginx:
+  package-name: nginx
+  version: "1.10.2-1.el7"
+nmap-ncat:
+  package-name: nmap-ncat
+  version: "6.40-7.el7"
+nodejs:
+  package-name: nodejs
+  version: "6.10.0-1nodesource.el7.centos"
+ntp:
+  package-name: ntp
+  version: "4.2.6p5-25.el7_3.1"
+opentsdb:
+  package-source: 'opentsdb-2.2.0.noarch.rpm'
+  bind_port: 4242
+python-pip:
+  package-name: python2-pip
+  version: "8.1.2-5.el7"
+python-dev:
+  package-name: python-devel
+  version: "2.7.5-48.el7"
+python3-pip:
+  package-name: python34-pip
+  version: "8.1.2-5.el7"
+python3-dev:
+  package-name: python34-devel
+  version: "3.4.5-3.el7"
+python-mysqldb:
+  package-name: MySQL-python
+  version: "1.2.5-1.el7"
+redis-server:
+  package-name: redis
+  version: "2.8.19-2.el7"
+  configuration_filename: "/etc/redis.conf"
+snappy:
+  package-name: snappy
+  version: "1.1.0-3.el7"
+sshfs:
+  package-name: fuse-sshfs
+  version: "2.5-1.el7"
+unzip:
+  package-name: unzip
+  version: "6.0-16.el7"
+wget:
+  package-name: wget
+  version: "1.14-13.el7"
diff --git a/pillar/packages/Ubuntu.sls b/pillar/packages/Ubuntu.sls
@@ -0,0 +1,86 @@
+acl:
+  package-name: acl
+  version: "2.2.52-1"
+cloudera-manager-agent:
+  package-name: cloudera-manager-agent
+  version: "5.9.0-1.cm590.p0.249~trusty-cm5"
+cloudera-manager-daemons:
+  package-name: cloudera-manager-daemons
+  version: "5.9.0-1.cm590.p0.249~trusty-cm5"
+cloudera-manager-server:
+  package-name: cloudera-manager-server
+  version: "5.9.0-1.cm590.p0.249~trusty-cm5"
+debconf-utils:
+  package-name: debconf-utils
+  version: "1.5.51ubuntu2"
+g++:
+  package-name: g++
+  version: "4:4.8.2-1ubuntu6"
+gnuplot:
+  package-name: gnuplot
+  version: "4.6.4-2"
+glibc-devel:
+  package-name: libc6-dev
+  version: "2.19-0ubuntu6.9"
+grafana:
+  package-source: 'grafana_3.1.1-1470047149_amd64.deb'
+libmysql-java:
+  package-name: libmysql-java
+  version: "5.1.28-1"
+libssl-dev:
+  package-name: libssl-dev
+  version: "1.0.1f-1ubuntu2.22"
+libffi-dev:
+  package-name: libffi-dev
+  version:  "3.1~rc1+r3.0.13-12ubuntu0.1"
+libsasl:
+  package-name: libsasl2-dev
+  version: "2.1.25.dfsg1-17build1"
+mysql-server:
+  package-name: mysql-server-5.6
+  version: "5.6.33-0ubuntu0.14.04.1"
+  configuration_file: /etc/mysql/my.cnf
+  service_name: mysql
+nginx:
+  package-name: nginx
+  version: "1.4.6-1ubuntu3.7"
+nodejs:
+  package-name: nodejs
+  version: "6.10.0-1nodesource1~trusty1"
+ntp:
+  package-name: ntp
+  version: "1:4.2.6.p5+dfsg-3ubuntu2.14.04.10"
+opentsdb:
+  package-source: 'opentsdb-2.2.0_all.deb'
+  bind_port: 4242
+python-mysqldb:
+  package-name: python-mysqldb
+  version: "1.2.3-2ubuntu1"
+python-pip:
+  package-name: python-pip
+  version: "1.5.4-1ubuntu4"
+python-dev:
+  package-name: python-dev
+  version: "2.7.5-5ubuntu3"
+python3-pip:
+  package-name: python3-pip
+  version: "1.5.4-1ubuntu4"
+python3-dev:
+  package-name: python3-dev
+  version: "3.4.0-0ubuntu2"
+redis-server:
+  package-name: redis-server
+  version: "2:2.8.4-2"
+  configuration_filename: "/etc/redis/redis.conf"
+snappy:
+  package-name: snappy
+  version: "0.2-1"
+sshfs:
+  package-name: sshfs
+  version: "2.5-1ubuntu1"
+unzip:
+  package-name: unzip
+  version: "6.0-9ubuntu1.5"
+wget:
+  package-name: wget
+  version: "1.15-1ubuntu1.14.04.2"
diff --git a/pillar/services.sls b/pillar/services.sls
@@ -55,15 +55,6 @@ kafkamanager:
 jupyterproxy:
   release_version: 1.3.1
 
-opentsdb:
-  version: 2.2.0
-  release_hash: sha256=e82738703efa50cfdd42dd7741e3d5b78fc2bf8cd12352253fc1489d1dea1f60
-  bind_port: 4242
-
-grafana:
-  version: 3.1.1-1470047149
-  release_hash: sha256=4d3153966afed9b874a6fa6182914d9bd2e69698bbc7c13248d1b7ef09d3d328
-
 gobblin:
   release_version: develop
 

diff --git a/pillar/top.sls b/pillar/top.sls
@@ -4,3 +4,4 @@
     - flavors.{{ salt['grains.get']('pnda:flavor', 'standard') }}
     - services
     - env_parameters
+    - packages.{{ grains['os'] }}
diff --git a/salt/cdh/cloudera-manager-agent.sls b/salt/cdh/cloudera-manager-agent.sls
@@ -3,8 +3,9 @@
 cloudera-manager-agent-install_packages:
   pkg.installed:
     - pkgs:
-      - cloudera-manager-daemons
-      - cloudera-manager-agent
+      - {{ pillar['cloudera-manager-daemons']['package-name'] }}: {{ pillar['cloudera-manager-daemons']['version'] }}
+      - {{ pillar['cloudera-manager-agent']['package-name'] }}: {{ pillar['cloudera-manager-agent']['version'] }}
+    - ignore_epoch: True
 
 cloudera-manager-agent-configure_ini:
   cmd.run:

diff --git a/salt/cdh/cloudera-manager.sls b/salt/cdh/cloudera-manager.sls
@@ -15,8 +15,8 @@ include:
 cloudera-manager-install_cloudera_manager:
   pkg.installed:
     - pkgs:
-      - cloudera-manager-daemons
-      - cloudera-manager-server
+      - {{ pillar['cloudera-manager-daemons']['package-name'] }}: {{ pillar['cloudera-manager-daemons']['version'] }}
+      - {{ pillar['cloudera-manager-server']['package-name'] }}: {{ pillar['cloudera-manager-server']['version'] }}
 
 {% if grains['os'] == 'RedHat' %}
 cloudera-manager-ensure_cloudera_manager_enabled:

diff --git a/salt/cdh/setup_hadoop.sls b/salt/cdh/setup_hadoop.sls
@@ -20,13 +20,9 @@ include:
 cdh-install_deps:
   pkg.installed:
     - pkgs:
-{% if grains['os'] == 'Ubuntu' %}
-      - libssl-dev
-      - libffi-dev
-{% elif grains['os'] == 'RedHat' %}
-      - libffi-devel
-      - openssl-devel
-{% endif %}
+      - {{ pillar['libffi-dev']['package-name'] }}: {{ pillar['libffi-dev']['version'] }}
+      - {{ pillar['libssl-dev']['package-name'] }}: {{ pillar['libssl-dev']['version'] }}
+    - ignore_epoch: True
 
 # Create a temporary virtualenv to execute the cm_setup scripts_location
 cdh-create_tmp_virtualenv:

diff --git a/salt/console-backend/data-logger.sls b/salt/console-backend/data-logger.sls
@@ -16,12 +16,9 @@ include:
 # Install nodejs, npm and redis-server
 console-backend-install_data_logger_redis:
   pkg.installed:
-    - pkgs:
-{% if grains['os'] == 'Ubuntu' %}
-      - redis-server
-{% elif grains['os'] == 'RedHat' %}
-      - redis
-{% endif %}
+    - name: {{ pillar['redis-server']['package-name'] }}
+    - version: {{ pillar['redis-server']['version'] }}
+    - ignore_epoch: True
 
 console-backend-dl-and-extract:
   archive.extracted:

diff --git a/salt/deployment-manager/init.sls b/salt/deployment-manager/init.sls
@@ -13,14 +13,12 @@ include:
 deployment-manager-install_dev_deps:
   pkg.installed:
     - pkgs:
-{% if grains['os'] == 'Ubuntu' %}
-      - libsasl2-dev
-      - g++
-{% elif grains['os'] == 'RedHat' %}
-      - gcc-c++
-      - libgsasl-devel
-      - cyrus-sasl-devel
+{% if grains['os'] == 'RedHat' %}
+      - {{ pillar['cyrus-sasl-devel']['package-name'] }}: {{ pillar['cyrus-sasl-devel']['version'] }}
 {% endif %}
+      - {{ pillar['libsasl']['package-name'] }}: {{ pillar['libsasl']['version'] }}
+      - {{ pillar['g++']['package-name'] }}: {{ pillar['g++']['version'] }}
+    - ignore_epoch: True
 
 deployment-manager-dl-and-extract:
   archive.extracted:
@@ -90,4 +88,4 @@ deployment-manager-systemctl_reload:
 
 deployment-manager-start_service:
   cmd.run:
-    - name: 'service deployment-manager stop || echo already stopped; service deployment-manager start'
+    - name: 'service deployment-manager stop || echo already stopped; service deployment-manager start'
diff --git a/salt/gnuplot/init.sls b/salt/gnuplot/init.sls
@@ -1,3 +1,5 @@
 gnuplot-install-gnuplot:
   pkg.installed:
-    - name: gnuplot
+    - name: {{ pillar['gnuplot']['package-name'] }}
+    - version: {{ pillar['gnuplot']['version'] }}
+    - ignore_epoch: True
diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls
@@ -5,14 +5,8 @@
 {% set misc_packages_path = pillar['pnda_mirror']['misc_packages_path'] %}
 {% set mirror_location = pnda_mirror + misc_packages_path %}
 
-{% set grafana_deb_package = 'grafana_' + grafana_version + '_amd64.deb' %}
-{% set grafana_rpm_package = 'grafana-' + grafana_version + '.x86_64.rpm' %}
-{% set grafana_deb_location = mirror_location + grafana_deb_package %}
-{% set grafana_rpm_location = mirror_location + grafana_rpm_package %}
-
 {% set pnda_graphite_port = 8013 %}
 {% set pnda_graphite_host = salt['pnda.ip_addresses']('graphite')[0] %}
-
 {% set datasources = [
     '{ "name": "PNDA OpenTSDB", "type": "opentsdb", "url": "http://localhost:4242", "access": "proxy", "basicAuth": false, "isDefault": true }',
     '{{ "name": "PNDA Graphite", "type": "graphite", "url": "http://{}:{}", "access": "proxy", "basicAuth": false, "isDefault": false }}'.format(pnda_graphite_host, pnda_graphite_port) ] %}
@@ -25,11 +19,7 @@
 grafana-server_pkg:
   pkg.installed:
     - sources:
-{% if grains['os'] == 'Ubuntu' %}
-      - grafana: {{ grafana_deb_location }}
-{% elif grains['os'] == 'RedHat' %}
-      - grafana: {{ grafana_rpm_location }}
-{% endif %}
+      - grafana: {{ mirror_location+pillar['grafana']['package-source'] }}
 
 {% if grains['os'] == 'RedHat' %}
 grafana-systemctl_reload:

diff --git a/salt/java/init.sls b/salt/java/init.sls
@@ -14,8 +14,9 @@
 
 java-install_dependencies:
   pkg.installed:
-    - pkgs:
-      - wget
+    - name: {{ pillar['wget']['package-name'] }}
+    - version: {{ pillar['wget']['version'] }}
+    - ignore_epoch: True
 
 {{ java.prefix }}:
   file.directory:

diff --git a/salt/java/jce.sls b/salt/java/jce.sls
@@ -10,7 +10,10 @@ include:
   - sun-java
 
 unzip:
-  pkg.installed
+  pkg.installed:
+    - name: {{ pillar['unzip']['package-name'] }}
+    - version: {{ pillar['unzip']['version'] }}
+    - ignore_epoch: True
 
 download-jce-zip:
   cmd.run:

diff --git a/salt/kafka-manager/init.sls b/salt/kafka-manager/init.sls
@@ -12,7 +12,9 @@
 
 kafka-manager-install_unzip:
   pkg.installed:
-    - name: unzip
+    - name: {{ pillar['unzip']['package-name'] }}
+    - version: {{ pillar['unzip']['version'] }}
+    - ignore_epoch: True
 
 kafka-manager-dl-and-extract:
   archive.extracted:

diff --git a/salt/logserver/logserver.sls b/salt/logserver/logserver.sls
@@ -13,19 +13,13 @@ include:
 
 install-redis_server:
   pkg.installed:
-{% if grains['os'] == 'Ubuntu' %}
-    - name: redis-server
-{% elif grains['os'] == 'RedHat' %}
-    - name: redis
-{% endif %}
+    - name: {{ pillar['redis-server']['package-name'] }}
+    - version: {{ pillar['redis-server']['version'] }}
+    - ignore_epoch: True
 
 change-bind-address_redis:
   file.replace:
-{% if grains['os'] == 'Ubuntu' %}
-    - name: /etc/redis/redis.conf
-{% elif grains['os'] == 'RedHat' %}
-    - name: /etc/redis.conf
-{% endif %}
+    - name: {{ pillar['redis-server']['configuration_filename'] }}
     - pattern: 'bind 127.0.0.1'
     - repl: 'bind 0.0.0.0'
 

diff --git a/salt/logserver/logshipper.sls b/salt/logserver/logshipper.sls
@@ -16,12 +16,9 @@ include:
 logshipper-lbc6:
   pkg.installed:
     - pkgs:
-      {% if grains['os'] == 'RedHat' %}
-      - glibc-devel
-      {% elif grains['os'] == 'Ubuntu' %}
-      - libc6-dev
-      {% endif %}
-      - acl
+      - {{ pillar['glibc-devel']['package-name'] }}: {{ pillar['glibc-devel']['version'] }}
+      - {{ pillar['acl']['package-name'] }}: {{ pillar['acl']['version'] }}
+    - ignore_epoch: True
 
 logshipper-dl-and-extract:
   archive.extracted: