Merge pull request #1 from misterbisson/auto-raft

Automatic Consul raft discovery and formation

Dockerfile

@@ -1,16 +1,48 @@
-FROM 		phusion/baseimage
-MAINTAINER 	Casey Bisson <[email protected]
+FROM alpine:3.2
+MAINTAINER 	Casey Bisson <[email protected]>
 
-# Update Apt
-RUN apt-get update
+# Alpine packages
+# Note: glibc is required because the Consul binary we're using is built against it
+RUN apk --update \
+    add \
+        jq \
+        curl \
+        bash \
+        ca-certificates && \
+    curl -Ls https://circle-artifacts.com/gh/andyshinn/alpine-pkg-glibc/6/artifacts/0/home/ubuntu/alpine-pkg-glibc/packages/x86_64/glibc-2.21-r2.apk > /tmp/glibc-2.21-r2.apk && \
+    apk add --allow-untrusted /tmp/glibc-2.21-r2.apk && \
+    rm -rf /tmp/glibc-2.21-r2.apk /var/cache/apk/*
 
-# Install some prereqs
-RUN apt-get install -y curl unzip npm
+# The Consul binary
+ADD https://dl.bintray.com/mitchellh/consul/0.5.2_linux_amd64.zip /tmp/consul.zip
+RUN cd /bin && \
+    unzip /tmp/consul.zip && \
+    chmod +x /bin/consul && \
+    rm /tmp/consul.zip
 
-# Get Consul
-ADD https://dl.bintray.com/mitchellh/consul/0.5.1_linux_amd64.zip /tmp/consul.zip
-RUN cd /bin && unzip /tmp/consul.zip && chmod +x /bin/consul && rm /tmp/consul.zip
+# The Consul web UI
+ADD https://dl.bintray.com/mitchellh/consul/0.5.2_web_ui.zip /tmp/webui.zip
+RUN mkdir /ui && \
+    cd /ui && \
+    unzip /tmp/webui.zip && \
+    rm /tmp/webui.zip && \
+    mv dist/* . && \
+    rm -rf dist
 
-# Get the Consul web UI
-ADD https://dl.bintray.com/mitchellh/consul/0.5.1_web_ui.zip /tmp/webui.zip
-RUN mkdir /ui && cd /ui && unzip /tmp/webui.zip && rm /tmp/webui.zip && mv dist/* . && rm -rf dist
+# Consul config
+COPY ./config /config/
+ONBUILD ADD ./config /config/
+
+COPY ./bin/* /bin/
+
+EXPOSE 8300 8301 8301/udp 8302 8302/udp 8400 8500 53 53/udp
+
+# Put Consul data on a separate volume to avoid filesystem performance issues with Docker image layers
+# Not necessary on Triton, but...
+VOLUME ["/data"]
+
+ENV GOMAXPROCS 2
+ENV SHELL /bin/bash
+
+ENTRYPOINT ["/bin/triton-start"]
+CMD []

LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2015 Casey Bisson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This project is built on Jeff Lindsay's substantial foundation work as found in https://github.com/gliderlabs/docker-consul/tree/legacy. The license for that work is included below.
+
+
+
+Copyright (C) 2014 Jeff Lindsay
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -1 +1,48 @@
-# Trusted, self-bootstrapping Consul
+# Triton trusted Consul
+
+[Consul](http://www.consul.io/) in Docker, designed for availability and durability.
+
+## Prep your environment
+
+1. [Get a Joyent account](https://my.joyent.com/landing/signup/) and [add your SSH key](https://docs.joyent.com/public-cloud/getting-started).
+1. Install and the [Docker Engine](https://docs.docker.com/installation/mac/) (including `docker` and `docker-compose`) on your laptop or other environment, along with the [Joyent CloudAPI CLI tools](https://apidocs.joyent.com/cloudapi/#getting-started) (including the `smartdc` and `json` tools).
+1. [Configure your Docker CLI and Compose for use with Joyent](https://docs.joyent.com/public-cloud/api-access/docker):
+
+```
+curl -O https://raw.githubusercontent.com/joyent/sdc-docker/master/tools/sdc-docker-setup.sh && chmod +x sdc-docker-setup.sh
+ ./sdc-docker-setup.sh -k us-east-1.api.joyent.com <ACCOUNT> ~/.ssh/<PRIVATE_KEY_FILE>
+```
+
+## Start a trusted Consul raft
+
+1. [Clone](https://github.com/misterbisson/triton-consul) or [download](https://github.com/misterbisson/triton-consul/archive/master.zip) this repo
+1. `cd` into the cloned or downloaded directory
+1. Execute `bash start.sh` to start everything up
+1. The Consul dashboard should automatically open in your browser, or follow the links output by the `start.sh` script
+
+## Use this in your own composition
+
+Detailed example to come....
+
+## How it works
+
+This demo actually sets up two independent Consul services:
+
+1. A single-node instance used only for bootstrapping the raft
+1. A three-node instance that other applications can point to
+
+A running raft has no dependency on the bootstrap instance. New raft instances do need to connect to the bootstrap instance to find the raft, creating a failure gap that is discussed below. If a raft instance fails, the data is preserved among the other instances and the overall availability of the service is preserved because any single instance can authoritatively answer for all instances. Applications that depend on the Consul service should re-try failed requests until they get a response.
+
+Each raft instance will constantly re-register with the bootstrap instance. If the boostrap instance or its data is lost, a new bootstrap instance can be started and all existing raft instances will re-register with it. In a scenario where the bootstrap instance is unavailable, it will be impossible to start raft instances until the bootstrap instance has been restarted and at least one existing raft member has reregistered.
+
+## Triston-specific availability advantages
+
+Some details about how Docker containers work on Triton have specific bearing on the durability and availability of this service:
+
+1. Docker containers are first-order objects on Triton. They run on bare metal, and their overall availability is similar or better than what you expect of a virtual machine in other environments.
+1. Docker containers on Triton preserve their IP and any data on disk when they reboot.
+1. Linked containers in Docker Compose on Triton are actually distributed across multiple unique physical nodes for maximum availability in the case of  node failures.
+
+# Credit where it's due
+
+This project builds on the fine examples set by [Jeff Lindsay](https://github.com/progrium)'s ([Glider Labs](https://github.com/gliderlabs)) [Consul in Docker](https://github.com/gliderlabs/docker-consul/tree/legacy) work. It also, obviously, wouldn't be possible without the outstanding work of the [Hashicorp team](https://hashicorp.com) that made [consul.io](https://www.consul.io).

bin/consul-health

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Consul heartbeat script
+# re-registers this node and sets it healthy in the non-HA Consul bootstrap service
+
+while true
+do
+    consul info &> /dev/null
+    if [ $? -eq 0 ]
+    then
+
+        # these executables get written by the triton-bootstrap script
+        bash /bin/consul-register-cmd
+        bash /bin/consul-health-cmd
+    fi
+
+    sleep $(( 10 + $RANDOM % 10 )) #sleep for 10-20 seconds
+done

bin/triton-bootstrap

@@ -0,0 +1,225 @@
+#!/bin/bash
+
+#
+# This is the startup script is run once on startup to find and join a Consul raft
+# it will continue polling for a raft until one is found
+#
+# The script can also be run with arguments to bootstrap the raft
+#
+
+# This container's IP(s)
+export IP_PRIVATE=$(ip addr show eth0 | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}')
+IP_HAVEPUBLIC=$(ip link show | grep eth1)
+if [[ $IP_HAVEPUBLIC ]]
+then
+    export IP_PUBLIC=$(ip addr show eth1 | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}')
+else
+    export IP_PUBLIC=$IP_PRIVATE
+fi
+
+# Discovery vars
+export CONSUL_SERVICE_NAME=${CONSUL_SERVICE_NAME:-haconsul}
+export BOOTSTRAP_HOST=${BOOTSTRAP_HOST:-'http://consulbootstrap:8500'}
+
+
+
+#
+# Write the Consul command and args to a file for use by the start script
+#
+consul_cmd()
+{
+    echo "/bin/consul agent -config-dir=/config $1" > /bin/consul-start-cmd
+
+    echo '#'
+    echo "# This node's Consul start command:"
+    echo '#'
+
+    cat /bin/consul-start-cmd
+    echo
+}
+
+
+
+#
+# Write the Consul registration and health check commands and args to a file for use by the start script
+#
+consul_register_cmd()
+{
+    echo curl -f --retry 7 --retry-delay 3 $BOOTSTRAP_HOST/v1/agent/service/register -d "'$(printf '{"ID": "%s-%s","Name": "%s","tags": ["consul"],"Address": "%s","checks": [{"ttl": "59s"}]}' $CONSUL_SERVICE_NAME $HOSTNAME $CONSUL_SERVICE_NAME $IP_PRIVATE)'" > /bin/consul-register-cmd
+
+    echo '#'
+    echo "# This node's Consul registration command:"
+    echo '#'
+
+    cat /bin/consul-register-cmd
+    echo
+}
+
+consul_health_cmd()
+{
+    echo curl -f --retry 7 --retry-delay 3 "'$BOOTSTRAP_HOST/v1/agent/check/pass/service:${CONSUL_SERVICE_NAME}-${HOSTNAME}?note=running+healthy'" > /bin/consul-health-cmd
+
+    echo '#'
+    echo "# This node's Consul health check update command:"
+    echo '#'
+
+    cat /bin/consul-health-cmd
+    echo
+}
+
+
+
+echo
+echo '#'
+echo '# Testing to see if Consul is already running'
+echo '#'
+
+consul info | grep server &> /dev/null
+if [ $? -eq 0 ]; then
+    echo
+    echo '#'
+    echo '# Already running as a server...'
+    echo '#'
+    echo "# Dashboard: http://$IP_PUBLIC:8500/ui"
+
+    exit
+fi
+
+
+
+echo
+echo '#'
+echo '# Checking bootstrap availability'
+echo '#'
+
+curl -fs --retry 7 --retry-delay 3 $BOOTSTRAP_HOST/v1/agent/services &> /dev/null
+if [ $? -ne 0 ]
+then
+    echo '# Ack!'
+    echo '# Bootstrap instance of Consul is required, but unreachable'
+    echo '#'
+    curl $BOOTSTRAP_HOST/v1/agent/services
+    exit
+else
+    echo '# Bootstrap instance found and responsive'
+    echo '#'
+fi
+
+
+
+#
+# Register this unconfigured Consul raft member wannabe in the bootstrap instance for discovery by other raft wannabees
+#
+curl -f --retry 7 --retry-delay 3 $BOOTSTRAP_HOST/v1/agent/service/register -d "$(printf '{"ID":"%s-unconfigured-%s","Name":"%s-unconfigured","Address":"%s","checks": [{"ttl": "59s"}]}' $CONSUL_SERVICE_NAME $HOSTNAME $CONSUL_SERVICE_NAME $IP_PRIVATE)"
+
+# pass the healthcheck
+curl -f --retry 7 --retry-delay 3 "$BOOTSTRAP_HOST/v1/agent/check/pass/service:$CONSUL_SERVICE_NAME-unconfigured-$HOSTNAME?note=initial+startup"
+
+
+
+#
+# Either bootstrap a new raft or poll for an existing raft
+#
+if [ "$1" = 'bootstrap' ]
+then
+    echo '#'
+    echo '# Bootstrapping raft'
+    echo '#'
+
+    #
+    # Deregister this raft wannabe from the list of unconfigured raft wannabees in the bootstrap node
+    #
+    curl -f --retry 7 --retry-delay 3 $BOOTSTRAP_HOST/v1/agent/service/deregister/$CONSUL_SERVICE_NAME-unconfigured-$HOSTNAME
+
+    echo
+    echo '#'
+    echo '# Bootstrapping the Consul raft...'
+    echo '#'
+    consul_cmd "-server -bootstrap -ui-dir /ui"
+
+else
+    echo '#'
+    echo '# Looking for an existing raft'
+    echo '#'
+
+    RAFTFOUND=0
+    RAFTIPREGEX='^[0-9]'
+    while [ $RAFTFOUND != 1 ]; do
+        echo -n '.'
+
+        RAFTIP=$(curl -L -s -f $BOOTSTRAP_HOST/v1/health/service/$CONSUL_SERVICE_NAME?passing | jq --raw-output '.[0] | .Service.Address')
+
+        if [[ $RAFTIP =~ $RAFTIPREGEX ]]
+        then
+            echo '#'
+            echo "# Raft found at $RAFTIP"
+            echo '#'
+
+            let RAFTFOUND=1
+        else
+            # Update the healthcheck for this unconfigured Consul raft wannabe in the bootstrap instance for discovery
+            curl -f --retry 7 --retry-delay 3 "$BOOTSTRAP_HOST/v1/agent/check/pass/service:$CONSUL_SERVICE_NAME-unconfigured-$HOSTNAME?note=polling+for+raft"
+
+            # sleep for a bit
+            sleep 7
+        fi
+    done
+
+    #
+    # Deregister this raft wannabe from the list of unconfigured raft wannabees in Consul
+    #
+    curl -f --retry 7 --retry-delay 3 $BOOTSTRAP_HOST/v1/agent/service/deregister/$CONSUL_SERVICE_NAME-unconfigured-$HOSTNAME
+
+    echo
+    echo '#'
+    echo '# Joining raft...'
+    echo '#'
+    consul_cmd "-server -join $RAFTIP -ui-dir /ui"
+fi
+
+
+
+echo
+echo '#'
+echo '# Confirming raft health...'
+echo '#'
+RESPONSIVE=0
+while [ $RESPONSIVE != 1 ]; do
+    echo -n '.'
+
+    consul info | grep server &> /dev/null
+    if [ $? -eq 0 ]
+    then
+        echo
+        echo '#'
+        echo '# Consul is running...'
+        echo '#'
+        echo "# Dashboard: http://$IP_PUBLIC:8500/ui"
+        echo '#'
+        consul info
+
+        let RESPONSIVE=1
+    else
+        sleep .7
+    fi
+done
+sleep 1
+
+
+
+echo
+echo '#'
+echo '# Register the Consul raft member'
+echo '#'
+curl -f --retry 7 --retry-delay 3 $BOOTSTRAP_HOST/v1/agent/service/register -d "$(printf '{"ID": "%s-%s","Name": "%s","tags": ["consul"],"Address": "%s","checks": [{"ttl": "59s"}]}' $CONSUL_SERVICE_NAME $HOSTNAME $CONSUL_SERVICE_NAME $IP_PRIVATE)"
+
+#
+# Write out the registration and health commands for use elsewhere
+#
+consul_register_cmd
+consul_health_cmd
+
+echo
+echo '#'
+echo '# Bootstrapping complete'
+echo '#'