intial baseline for EDB support

.gitignore

@@ -0,0 +1,6 @@
+# ignore all private keys
+*.pem
+ansible.log
+.DS_Store
+*.retry
+.python-version

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "common-roles"]
+	path = common-roles
+	url = https://github.com/Datanexus/common-roles.git
+[submodule "common-utils"]
+	path = common-utils
+	url = https://github.com/Datanexus/common-utils.git

.tm_properties

@@ -0,0 +1,2 @@
+[ "{provision-edb}" ]
+fileType = source.python

CODEOWNERS

@@ -0,0 +1,13 @@
+# Lines starting with '#' are comments.
+# Each line is a file pattern followed by one or more owners.
+
+# These owners will be the default owners for everything in the repo.
+*       @christopherkeller
+
+# Order is important. The last matching pattern has the most precedence.
+# So if a pull request only touches javascript files, only these owners
+# will be requested to review.
+# *.js    @octocat @github/js
+
+# You can also use email addresses if you prefer.
+# docs/*  [email protected]

CONTRIBUTING.md

@@ -0,0 +1,14 @@
+## Contributing to dn-postgresql
+
+*Before opening a pull request* consider:
+
+- Is the change important and ready enough to ask the community to spend time reviewing?
+- Have you searched for existing, related GitHub issues and pull requests?
+- Is this a new feature that can stand alone?
+- Is the change being proposed clearly explained and motivated?
+
+When you contribute code, you affirm that the contribution is your original work and that you 
+license the work to the project under the project's open source license. Whether or not you 
+state this explicitly, by submitting any copyrighted material via pull request, email, or 
+other means you agree to license the material under the project's open source license and 
+warrant that you have the legal authority to do so.

README.md

@@ -1,2 +1,92 @@
-# edb
-EDB commercial database
+© 2016 DataNexus Inc.
+
+Role Name
+=========
+
+Builds a stand-alone postgresql database with SSL connectivity. If multiple servers have been provisioned and tagged with "Role:master" and "Role:replica" then a streaming master-replica will be provisioned.
+
+Role Variables
+--------------
+
+    application: postgresql
+    postgresql_version: "9.2.18-1.el7"
+    python_postgres_adapter_version: "2.5.1-3.el7"
+    postgresql_package_list:
+      - "python-psycopg2-{{ python_postgres_adapter_version }}"
+      - "postgresql-{{ postgresql_version }}"
+      - "postgresql-server-{{ postgresql_version }}"
+      - "postgresql-contrib-{{ postgresql_version }}"
+      - "postgresql-libs-{{ postgresql_version }}"
+    postgresql_data_dir: "/data/pgsql"
+    postgresql_home_dir: "/var/lib/pgsql"
+    postgresql_bin_path: "/usr/bin"
+    postgresql_config_path: "{{ postgresql_data_dir }}"
+    postgresql_daemon: postgresql
+    postgresql_interface: eth1
+    ansible_ssh_private_key_file: "aws_{{ hostvars[inventory_hostname].ec2_key_name }}_private_key.pem"
+
+Dependencies
+------------
+Instances must be tagged: 
+
+    Cloud
+    Tenant
+    Project
+    Domain
+    Application
+    [Role]
+
+Playbook
+----------------
+The _site.yml_ playbook contains the necessary code to provision either an individual server or a master-replica pair.
+
+Call like the  playbook like this:
+
+    AWS_PROFILE=PROFILE ansible-playbook -e "project=PROJECT application=postgresql domain=DOMAIN  host_inventory=tag_Application_{{ application }} ansible_user=USER" site.yml
+
+where:
+
+    PROFILE is your ~/.aws/credentials profile
+
+and
+
+    PROJECT is the project name
+    DOMAIN is the domain: development or production or similar
+    USER is the instance login, ec2-user, redhat, centos, etc
+
+The command that was used  during development and testing on OSP was:
+
+    AWS_PROFILE=datanexus ./provision-postgresql -i inventory -e "cloud=osp ec2_region=regionOne application=postgresql domain=development project=demo tenant=dev key_path=/tmp ansible_user=cloud-user tenant_config_path=../"
+
+Testing
+----------------
+To verify that both SSL and replication are working run the test playbook:
+
+    AWS_PROFILE=datanexus ansible-playbook -e "project=demo application=postgresql domain=development  host_inventory=tag_Application_{{ application }} ansible_user=centos" test.yml
+
+Log into the replica node via SSH and verify the test table was replicated.
+
+     $ sudo -i -u postgres psql -c "select count(*) from t_random"
+     count
+    -------
+       500
+    (1 row)
+
+To verify SSL, from the replica machine log into MASTER_IP using the replicator role and postgres database :
+
+    $ sudo -i -u postgres psql -h MASTER_IP -U replicator postgres
+    psql (9.2.18)
+    SSL connection (cipher: DHE-RSA-AES256-GCM-SHA384, bits: 256)
+    Type "help" for help.
+
+    postgres=>
+
+License
+-------
+
+Apache
+
+Author Information
+------------------
+
+[Christopher Keller @ DataNexus ](mailto:[email protected])

ansible.cfg

@@ -0,0 +1,6 @@
+# (c) 2016 DataNexus Inc.  All Rights Reserved.
+
+[defaults]
+roles_path = ../:common-roles
+host_key_checking = False
+pipelining = False

defaults/main.yml

@@ -0,0 +1,46 @@
+---
+# RHEL/CentOS only. Set a repository to use for PostgreSQL installation.
+postgresql_enablerepo: ""
+
+postgresql_user: postgres
+postgresql_group: postgres
+
+postgresql_unix_socket_directories:
+  - /var/run/postgresql
+
+# Global configuration options that will be set in postgresql.conf.
+postgresql_global_config_options:
+  - option: unix_socket_directories
+    value: '{{ postgresql_unix_socket_directories | join(",") }}'
+
+# Debian only. Used to generate the locales used by PostgreSQL databases.
+postgresql_locales:
+  - 'en_US.UTF-8'
+
+# Databases to ensure exist.
+postgresql_databases: []
+  # - name: exampledb # required; the rest are optional
+  #   lc_collate: # defaults to 'en_US.UTF-8'
+  #   lc_ctype: # defaults to 'en_US.UTF-8'
+  #   encoding: # defaults to 'UTF-8'
+  #   template: # defaults to 'template0'
+  #   login_host: # defaults to 'localhost'
+  #   login_password: # defaults to not set
+  #   login_user: # defaults to '{{ postgresql_user }}'
+  #   login_unix_socket: # defaults to 1st of postgresql_unix_socket_directories
+  #   port: # defaults to not set
+  #   state: # defaults to 'present'
+
+# Users to ensure exist
+postgresql_users: []
+#  - name: #required; the rest are optional
+#    password: # defaults to not set
+#    priv: # defaults to not set
+#    role_attr_flags: # defaults to not set
+#    db: # defaults to not set
+#    login_host: # defaults to 'localhost'
+#    login_password: # defaults to not set
+#    login_user: # defaults to '{{ postgresql_user }}'
+#    login_unix_socket: # defaults to 1st of postgresql_unix_socket_directories
+#    port: # defaults to not set
+#    state: # defaults to 'present

handlers/main.yml

@@ -0,0 +1,8 @@
+# (c) Copyright 2016 DataNexus Inc.  All Rights Reserved. 
+#
+# simple handlers for restarting services
+---  
+- name: restart postgresql
+  become: yes
+  systemd: "name={{ postgresql_daemon }} state=restarted daemon_reload=yes"
+

master-streaming/defaults/main.yml

@@ -0,0 +1,21 @@
+# (c) Copyright 2016 DataNexus Inc.  All Rights Reserved. 
+#
+# streaming roles and tasks for the master
+---
+postgresql_user: postgres
+postgresql_unix_socket_directories:
+  - /var/run/postgresql
+
+# add replicator user by default
+replication_users:
+  - name: replicator #required; the rest are optional
+#    password: # defaults to not set
+#    priv: # defaults to not set
+    role_attr_flags: Replication # defaults to not set
+#    db: # defaults to not set
+#    login_host: # defaults to 'localhost'
+#    login_password: # defaults to not set
+#    login_user: # defaults to '{{ postgresql_user }}'
+#    login_unix_socket: # defaults to 1st of postgresql_unix_socket_directories
+#    port: # defaults to not set
+#    state: # defaults to 'present

master-streaming/handlers/main.yml

@@ -0,0 +1,8 @@
+# (c) Copyright 2016 DataNexus Inc.  All Rights Reserved. 
+#
+# simple handlers for restarting services
+---  
+- name: restart postgresql
+  become: yes
+  systemd: "name={{ postgresql_daemon }} state=restarted daemon_reload=yes"
+

master-streaming/tasks/main.yml

@@ -0,0 +1,70 @@
+# (c) Copyright 2016 DataNexus Inc.  All Rights Reserved. 
+#
+# streaming tasks for the master
+---
+#- set_fact: replica_server={{ hostvars[inventory_hostname].groups.tag_Role_replica[0] }}
+#  when: hostvars[inventory_hostname].groups.tag_Role_replica[0] is defined
+
+- block:
+  # this needs to be the data network on eth1  
+  - name: configure replication user
+    lineinfile:
+      dest: "{{ postgresql_config_path }}/pg_hba.conf"
+      line: "hostssl  replication     replicator      {{ hostvars[item].ansible_eth1.ipv4.address }}/32          cert clientcert=1"
+    become: true
+    with_items: "{{ groups.postgresql_replica }}"
+    notify: restart postgresql  
+
+  - name: adding replication user
+    postgresql_user:
+      name: "{{ item.name }}"
+      password: "{{ item.password | default(omit) }}"
+      priv: "{{ item.priv | default(omit) }}"
+      role_attr_flags: "{{ item.role_attr_flags | default(omit) }}"
+      db: "{{ item.db | default(omit) }}"
+      login_host: "{{ item.login_host | default('localhost') }}"
+      login_password: "{{ item.login_password | default(omit) }}"
+      login_user: "{{ item.login_user | default(postgresql_user) }}"
+      login_unix_socket: "{{ item.login_unix_socket | default(postgresql_unix_socket_directories[0]) }}"
+      port: "{{ item.port | default(omit) }}"
+      state: "{{ item.state | default('present') }}"
+    with_items: "{{ replication_users  }}"
+    no_log: true
+    become_user: "{{ postgresql_user }}"
+
+  - name: configure write ahead log for streaming
+    lineinfile:
+      dest: "{{ postgresql_config_path }}/postgresql.conf"
+      regexp: "^#wal_level"
+      backrefs: yes
+      line: "wal_level = hot_standby\t\t\t# minimal, archive, or hot_standby"
+    notify: restart postgresql
+
+  # max_wal_senders should be four times the total number of replicas 
+  - set_fact: wal_senders={{ groups['postgresql_replica'] | length * 4 }}  
+
+  - name: configure write ahead log senders
+    lineinfile:
+      dest: "{{ postgresql_config_path }}/postgresql.conf"
+      regexp: "^#max_wal_senders"
+      backrefs: yes
+      line: "max_wal_senders = {{ wal_senders }} \t\t# max number of walsender processes"
+    notify: restart postgresql
+
+  - name: configure checkpoint segments
+    lineinfile:
+      dest: "{{ postgresql_config_path }}/postgresql.conf"
+      regexp: "^#checkpoint_segments"
+      backrefs: yes
+      line: "checkpoint_segments = 8\t\t\t# in logfile segments, min 1, 16MB each"
+    notify: restart postgresql
+
+  - name: configure write ahead log keep segments
+    lineinfile:
+      dest: "{{ postgresql_config_path }}/postgresql.conf"
+      regexp: "^#wal_keep_segments"
+      backrefs: yes
+      line: "wal_keep_segments = 8\t\t# in logfile segments, 16MB each; 0 disables"
+    notify: restart postgresql
+  when: inventory_hostname in groups.postgresql_master
+  become: true