first commit

.gitattributes

@@ -0,0 +1 @@
+* text=auto

.gitignore

@@ -0,0 +1 @@
+.idea

CHANGELOG

@@ -0,0 +1,6 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+
+## [0.1.0] 2016-07-01
+### First version
+- Creates and configures all PNDA services

LICENSE

@@ -0,0 +1,15 @@
+Unless otherwise specified in the file, this software is:
+
+Copyright (c) 2016 Cisco and/or its affiliates.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

@@ -0,0 +1,120 @@
+# Salt Master
+
+To setup salt master (change address):
+
+Requirements:
+
+```sh
+sudo apt-get update
+sudo apt-get -y install python-pip python-git
+sudo pip install netaddr apache-libcloud
+```
+
+Then download salt installer and install it
+
+```sh
+wget -O install_salt.sh https://bootstrap.saltstack.com
+sudo sh install_salt.sh -D -U -M stable 2015.8.10
+```
+
+Replace /etc/salt/master with the following content:
+
+```yaml
+## specific PNDA saltmaster config
+auto_accept: True      # auto accept minion key on new minion provisioning
+
+## Using GitFS backend
+fileserver_backend:
+  - git
+  - minion
+
+gitfs_remotes:
+  - file:///home/cloud-user/git_repos/saltmaster.git:
+    - root: salt
+
+gitfs_base: master
+
+# Do not merge top.sls files across multiple environments
+top_file_merging_strategy: same
+
+ext_pillar:
+   - git: __env__ file:///home/cloud-user/git_repos/saltmaster.git root=pillar
+
+# To autoload new created modules, states add and remove salt keys,
+# update bastion /etc/hosts file automatically ... add the following reactor configuration
+reactor:
+  - 'minion_start':
+    - salt://reactor/sync_all.sls
+  - 'salt/cloud/*/created':
+    - salt://reactor/create_bastion_host_entry.sls
+  - 'salt/cloud/*/destroying':
+    - salt://reactor/delete_bastion_host_entry.sls
+
+file_recv: True
+
+failhard: True
+
+## end of specific PNDA saltmaster config
+```
+
+Then save and restart salt master process
+
+```sh
+sudo restart salt-master
+```
+
+In order for the salt/cloud events to work, you need to configure salt-cloud inside salt. It means, having `/etc/salt/cloud` and `/etc/salt/cloud.{providers,profiles}.d` configuration files.
+
+## Git repo for Salt recipes
+
+On the saltmaster instance, setup git repos (need to be done once and only once):
+
+```sh
+mkdir /home/cloud-user/git_repos
+cd /home/cloud-user/git_repos
+for r in salt-cloud.git saltmaster.git; do
+  mkdir $r
+  cd $r
+  git --bare init
+  cd -
+done
+```
+
+## Git repo initialization
+
+For the following git repos from the pnda git server:
+
+- platform-salt
+- platform-salt-cloud
+
+On a developer workstation, after cloning the said repos, perform the following steps where `saltmaster` refers to the remote saltmaster server:
+
+
+
+```
+cd <local-path>/platform-salt
+```
+```
+git remote add saltmaster ssh://cloud-user@saltmaster/home/cloud-user/git_repos/saltmaster.git
+```
+```
+cd <local-path>/platform-salt-cloud
+```
+```
+git remote add saltmaster ssh://cloud-user@saltmaster/home/cloud-user/git_repos/salt-cloud.git
+```
+
+So that once making updates, you will need to put them both on the pnda git server and the saltmaster git remotes
+
+```sh
+git push saltmaster master
+```
+
+## Miscellaneous (salt-master)
+
+Setup access to repo from local (need to be done once)
+
+```sh
+cd /home/cloud-user/johndoe
+git clone file:///home/saltgit/saltmaster.git
+```

pillar/env_parameters.sls

@@ -0,0 +1,2 @@
+# This file contains parameters inserted at provisioning time
+# Do not edit!

pillar/pnda.sls

@@ -0,0 +1,10 @@
+pnda:
+  user: 'pnda'
+  password_hash: $6$g4LehAEw$gIGXrIXeQBXgytGswU3m1Ovtb7FpGMZpWd5P8wdrbaGXFN.HeJ1UE1Hp/d6jAbEmTuymdcQAhEKQlDxd53Gjn1
+  password: pnda
+  group: 'pnda'
+
+  master_dataset:
+    directory: /user/pnda/PNDA_datasets/datasets
+    quarantine_directory: /user/pnda/PNDA_datasets/quarantine
+

pillar/services.sls

@@ -0,0 +1,105 @@
+mine_functions:
+  network.ip_addrs: [eth0]
+  grains.items: []
+
+logstash:
+  version: 1.4.2
+  directory: /opt/pnda
+
+kibana:
+  version: 4.1.6-linux-x64
+  directory: /opt/pnda
+
+elasticsearch:
+  version: 1.5.0
+  directory: /opt/pnda
+  logdir: /var/log/elasticsearch
+  datadir: /var/lib/elasticsearch
+  confdir: /etc/elasticsearch
+  workdir: /tmp/elasticsearch
+
+jupyter:
+  version: 4.2.0
+  confdir: /usr/local/etc/jupyter
+  kerneldir: /usr/local/share/jupyter/kernels
+
+jupyterhub:
+  version: 0.6.1
+  confdir: /etc/jupyterhub
+
+kafka:
+  version: 0.9.0.1
+  config:
+    log_dirs:
+      - '/var/kafka-logs'
+
+kafkacontrib:
+  release_directory: /opt/pnda
+  release_version: logstash-1.4.2-contrib-kafka-0.7.5
+
+admin_login:
+  user: admin
+  password: admin
+
+kafkamanager:
+  release_directory: /opt/pnda
+  release_version: 1.3.0.8
+
+opentsdb:
+  version: 2.2.0RC1
+  release_hash: sha256=199f60f31c8f72948d0e5a2c4695aedcb114360a77c4246b16587f07028f8068
+
+grafana:
+  version: 2.1.3
+  release_hash: sha256=7142e7239de5357e3769a286cd3b0c2c63a36234d30516ba9b96e7d088ece5bc
+
+gobblin:
+  release_version: 3.0.0
+
+console_frontend:
+  release_version: 3.0.0
+
+console_backend_data_logger:
+  release_version: 3.0.0 
+  bind_port: 3001
+
+console_backend_data_manager:
+  release_version: 3.0.0
+  bind_port: 3123
+
+deployment_manager:
+  release_version: 3.0.0
+
+package_repository:
+  release_version: 3.0.0
+
+data-service:
+  release_version: 3.0.0
+
+hdfs_cleaner:
+  release_version: 3.0.0
+
+platform_testing:
+  release_directory: /opt/pnda
+  release_version: 3.0.0
+
+ntp:
+  servers:
+    - "ntp.esl.cisco.com iburst"
+
+platformlib:
+  release_version: "3.0.0"
+  target_directory: "/opt/pnda"
+
+nginx:
+  admin_user: admin
+  admin_password: admin
+  certificates_email: [email protected]
+
+jmxproxy:
+  release_version: "3.2.0"
+  release_hash: "sha512=97e69d7922f6515bc5ecaa9ab7326e2d61d275dd8d419bdb2fb246ec36dbc21cb8df45881a991623f1a8785744a618198094f16f37d5b66f3029516d8824b7a1"
+
+anaconda:
+  parcel_version: "4.0.0"
+  parcel_repo: "https://repo.continuum.io/pkgs/misc/parcels/"

pillar/top.sls

@@ -0,0 +1,5 @@
+{{ env }}:
+  '*':
+    - pnda
+    - services
+    - env_parameters