diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/Dockerfile b/Dockerfile old mode 100755 new mode 100644 index 8297972..5e64e95 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,6 @@ MAINTAINER Casey Bisson # Note: glibc is required because the Consul binary we're using is built against it RUN apk --update \ add \ - jq \ curl \ bash \ ca-certificates && \ @@ -29,10 +28,20 @@ RUN mkdir /ui && \ mv dist/* . && \ rm -rf dist +# get Containerbuddy release +RUN export CB=containerbuddy-0.0.1-alpha &&\ + mkdir -p /opt/containerbuddy && \ + curl -Lo /tmp/${CB}.tar.gz \ + https://github.com/joyent/containerbuddy/releases/download/0.0.1-alpha/${CB}.tar.gz && \ + tar xzf /tmp/${CB}.tar.gz -C /tmp && \ + mv /tmp/build/containerbuddy /opt/containerbuddy/ +COPY containerbuddy.json /etc/ + # Consul config COPY ./config /config/ ONBUILD ADD ./config /config/ +# copy bootstrap scripts COPY ./bin/* /bin/ EXPOSE 8300 8301 8301/udp 8302 8302/udp 8400 8500 53 53/udp @@ -43,6 +52,3 @@ VOLUME ["/data"] ENV GOMAXPROCS 2 ENV SHELL /bin/bash - -ENTRYPOINT ["/bin/triton-start"] -CMD [] diff --git a/LICENSE b/LICENSE old mode 100755 new mode 100644 diff --git a/README.md b/README.md old mode 100755 new mode 100644 index e0a8659..b518825 --- a/README.md +++ b/README.md @@ -26,16 +26,13 @@ Detailed example to come.... ## How it works -This demo actually sets up two independent Consul services: +This demo first starts up a bootstrap node that starts the raft but expects 2 additional nodes before the raft is healthy. Once this node is up and its IP address is obtained, the rest of the nodes are started and joined to the bootstrap IP address (the value is passed in the `BOOTSTRAP_HOST` environment variable). -1. A single-node instance used only for bootstrapping the raft -1. A three-node instance that other applications can point to +If a raft instance fails, the data is preserved among the other instances and the overall availability of the service is preserved because any single instance can authoritatively answer for all instances. Applications that depend on the Consul service should re-try failed requests until they get a response. -A running raft has no dependency on the bootstrap instance. New raft instances do need to connect to the bootstrap instance to find the raft, creating a failure gap that is discussed below. If a raft instance fails, the data is preserved among the other instances and the overall availability of the service is preserved because any single instance can authoritatively answer for all instances. Applications that depend on the Consul service should re-try failed requests until they get a response. +Any new raft instances need to be started with a bootstrap IP address, but after the initial cluster is created, the `BOOTSTRAP_HOST` IP address can be any host currently in the raft. This means there is no dependency on the first node after the cluster has been formed. -Each raft instance will constantly re-register with the bootstrap instance. If the boostrap instance or its data is lost, a new bootstrap instance can be started and all existing raft instances will re-register with it. In a scenario where the bootstrap instance is unavailable, it will be impossible to start raft instances until the bootstrap instance has been restarted and at least one existing raft member has reregistered. - -## Triston-specific availability advantages +## Triton-specific availability advantages Some details about how Docker containers work on Triton have specific bearing on the durability and availability of this service: @@ -45,4 +42,4 @@ Some details about how Docker containers work on Triton have specific bearing on # Credit where it's due -This project builds on the fine examples set by [Jeff Lindsay](https://github.com/progrium)'s ([Glider Labs](https://github.com/gliderlabs)) [Consul in Docker](https://github.com/gliderlabs/docker-consul/tree/legacy) work. It also, obviously, wouldn't be possible without the outstanding work of the [Hashicorp team](https://hashicorp.com) that made [consul.io](https://www.consul.io). \ No newline at end of file +This project builds on the fine examples set by [Jeff Lindsay](https://github.com/progrium)'s ([Glider Labs](https://github.com/gliderlabs)) [Consul in Docker](https://github.com/gliderlabs/docker-consul/tree/legacy) work. It also, obviously, wouldn't be possible without the outstanding work of the [Hashicorp team](https://hashicorp.com) that made [consul.io](https://www.consul.io). diff --git a/bin/consul-health b/bin/consul-health deleted file mode 100755 index 440f858..0000000 --- a/bin/consul-health +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Consul heartbeat script -# re-registers this node and sets it healthy in the non-HA Consul bootstrap service - -while true -do - consul info &> /dev/null - if [ $? -eq 0 ] - then - - # these executables get written by the triton-bootstrap script - bash /bin/consul-register-cmd - bash /bin/consul-health-cmd - fi - - sleep $(( 10 + $RANDOM % 10 )) #sleep for 10-20 seconds -done diff --git a/bin/consul-heartbeat b/bin/consul-heartbeat new file mode 100755 index 0000000..b72ea28 --- /dev/null +++ b/bin/consul-heartbeat @@ -0,0 +1,25 @@ +#!/bin/bash + +# 1. check if member of a raft +# 2a. yes? --> execute health check +# 2b. no? --> check if BOOTSTRAP_HOST is set? +# 3a. no? --> join self +# 3b. yes? --> join raft + +log() { + echo " $(date -u '+%Y-%m-%d %H:%M:%S') containerbuddy: $@" +} + +if [ $(consul info | awk '/num_peers/{print $3}') == 0 ]; then + log "No peers in raft" + if [ -n "${BOOTSTRAP_HOST}" ]; then + log "Joining raft at ${BOOTSTRAP_HOST}" + consul join ${BOOTSTRAP_HOST} + else + log "Bootstrapping raft with self" + consul join $(ip addr show eth0 | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}') + fi +else + consul info &> /dev/null + exit $? +fi diff --git a/bin/triton-bootstrap b/bin/triton-bootstrap deleted file mode 100755 index 73c2cab..0000000 --- a/bin/triton-bootstrap +++ /dev/null @@ -1,225 +0,0 @@ -#!/bin/bash - -# -# This is the startup script is run once on startup to find and join a Consul raft -# it will continue polling for a raft until one is found -# -# The script can also be run with arguments to bootstrap the raft -# - -# This container's IP(s) -export IP_PRIVATE=$(ip addr show eth0 | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}') -IP_HAVEPUBLIC=$(ip link show | grep eth1) -if [[ $IP_HAVEPUBLIC ]] -then - export IP_PUBLIC=$(ip addr show eth1 | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}') -else - export IP_PUBLIC=$IP_PRIVATE -fi - -# Discovery vars -export CONSUL_SERVICE_NAME=${CONSUL_SERVICE_NAME:-haconsul} -export BOOTSTRAP_HOST=${BOOTSTRAP_HOST:-'http://consulbootstrap:8500'} - - - -# -# Write the Consul command and args to a file for use by the start script -# -consul_cmd() -{ - echo "/bin/consul agent -config-dir=/config $1" > /bin/consul-start-cmd - - echo '#' - echo "# This node's Consul start command:" - echo '#' - - cat /bin/consul-start-cmd - echo -} - - - -# -# Write the Consul registration and health check commands and args to a file for use by the start script -# -consul_register_cmd() -{ - echo curl -f --retry 7 --retry-delay 3 $BOOTSTRAP_HOST/v1/agent/service/register -d "'$(printf '{"ID": "%s-%s","Name": "%s","tags": ["consul"],"Address": "%s","checks": [{"ttl": "59s"}]}' $CONSUL_SERVICE_NAME $HOSTNAME $CONSUL_SERVICE_NAME $IP_PRIVATE)'" > /bin/consul-register-cmd - - echo '#' - echo "# This node's Consul registration command:" - echo '#' - - cat /bin/consul-register-cmd - echo -} - -consul_health_cmd() -{ - echo curl -f --retry 7 --retry-delay 3 "'$BOOTSTRAP_HOST/v1/agent/check/pass/service:${CONSUL_SERVICE_NAME}-${HOSTNAME}?note=running+healthy'" > /bin/consul-health-cmd - - echo '#' - echo "# This node's Consul health check update command:" - echo '#' - - cat /bin/consul-health-cmd - echo -} - - - -echo -echo '#' -echo '# Testing to see if Consul is already running' -echo '#' - -consul info | grep server &> /dev/null -if [ $? -eq 0 ]; then - echo - echo '#' - echo '# Already running as a server...' - echo '#' - echo "# Dashboard: http://$IP_PUBLIC:8500/ui" - - exit -fi - - - -echo -echo '#' -echo '# Checking bootstrap availability' -echo '#' - -curl -fs --retry 7 --retry-delay 3 $BOOTSTRAP_HOST/v1/agent/services &> /dev/null -if [ $? -ne 0 ] -then - echo '# Ack!' - echo '# Bootstrap instance of Consul is required, but unreachable' - echo '#' - curl $BOOTSTRAP_HOST/v1/agent/services - exit -else - echo '# Bootstrap instance found and responsive' - echo '#' -fi - - - -# -# Register this unconfigured Consul raft member wannabe in the bootstrap instance for discovery by other raft wannabees -# -curl -f --retry 7 --retry-delay 3 $BOOTSTRAP_HOST/v1/agent/service/register -d "$(printf '{"ID":"%s-unconfigured-%s","Name":"%s-unconfigured","Address":"%s","checks": [{"ttl": "59s"}]}' $CONSUL_SERVICE_NAME $HOSTNAME $CONSUL_SERVICE_NAME $IP_PRIVATE)" - -# pass the healthcheck -curl -f --retry 7 --retry-delay 3 "$BOOTSTRAP_HOST/v1/agent/check/pass/service:$CONSUL_SERVICE_NAME-unconfigured-$HOSTNAME?note=initial+startup" - - - -# -# Either bootstrap a new raft or poll for an existing raft -# -if [ "$1" = 'bootstrap' ] -then - echo '#' - echo '# Bootstrapping raft' - echo '#' - - # - # Deregister this raft wannabe from the list of unconfigured raft wannabees in the bootstrap node - # - curl -f --retry 7 --retry-delay 3 $BOOTSTRAP_HOST/v1/agent/service/deregister/$CONSUL_SERVICE_NAME-unconfigured-$HOSTNAME - - echo - echo '#' - echo '# Bootstrapping the Consul raft...' - echo '#' - consul_cmd "-server -bootstrap -ui-dir /ui" - -else - echo '#' - echo '# Looking for an existing raft' - echo '#' - - RAFTFOUND=0 - RAFTIPREGEX='^[0-9]' - while [ $RAFTFOUND != 1 ]; do - echo -n '.' - - RAFTIP=$(curl -L -s -f $BOOTSTRAP_HOST/v1/health/service/$CONSUL_SERVICE_NAME?passing | jq --raw-output '.[0] | .Service.Address') - - if [[ $RAFTIP =~ $RAFTIPREGEX ]] - then - echo '#' - echo "# Raft found at $RAFTIP" - echo '#' - - let RAFTFOUND=1 - else - # Update the healthcheck for this unconfigured Consul raft wannabe in the bootstrap instance for discovery - curl -f --retry 7 --retry-delay 3 "$BOOTSTRAP_HOST/v1/agent/check/pass/service:$CONSUL_SERVICE_NAME-unconfigured-$HOSTNAME?note=polling+for+raft" - - # sleep for a bit - sleep 7 - fi - done - - # - # Deregister this raft wannabe from the list of unconfigured raft wannabees in Consul - # - curl -f --retry 7 --retry-delay 3 $BOOTSTRAP_HOST/v1/agent/service/deregister/$CONSUL_SERVICE_NAME-unconfigured-$HOSTNAME - - echo - echo '#' - echo '# Joining raft...' - echo '#' - consul_cmd "-server -join $RAFTIP -ui-dir /ui" -fi - - - -echo -echo '#' -echo '# Confirming raft health...' -echo '#' -RESPONSIVE=0 -while [ $RESPONSIVE != 1 ]; do - echo -n '.' - - consul info | grep server &> /dev/null - if [ $? -eq 0 ] - then - echo - echo '#' - echo '# Consul is running...' - echo '#' - echo "# Dashboard: http://$IP_PUBLIC:8500/ui" - echo '#' - consul info - - let RESPONSIVE=1 - else - sleep .7 - fi -done -sleep 1 - - - -echo -echo '#' -echo '# Register the Consul raft member' -echo '#' -curl -f --retry 7 --retry-delay 3 $BOOTSTRAP_HOST/v1/agent/service/register -d "$(printf '{"ID": "%s-%s","Name": "%s","tags": ["consul"],"Address": "%s","checks": [{"ttl": "59s"}]}' $CONSUL_SERVICE_NAME $HOSTNAME $CONSUL_SERVICE_NAME $IP_PRIVATE)" - -# -# Write out the registration and health commands for use elsewhere -# -consul_register_cmd -consul_health_cmd - -echo -echo '#' -echo '# Bootstrapping complete' -echo '#' diff --git a/bin/triton-start b/bin/triton-start deleted file mode 100755 index f1beb5c..0000000 --- a/bin/triton-start +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# Triton-optimized Consul start script. - -case "$1" in - - # - # This is the normal case for an HA raft of Consul instances - # - raft) - # Start the bootstrap script and background it - # the script will wait for for a raft to form, generate the command to start Consul, then exit - /bin/triton-bootstrap & - - # wait for the execution command to be generated by the bootstrap script - CONSULSCRIPT=/bin/consul-start-cmd - while [ ! -e $CONSULSCRIPT ] - do - echo -n '.' - sleep .7 - done - - # sleep another moment - sleep .7 - - # start the heartbeat script in the background - /bin/consul-health & - - # Start Consul - bash $CONSULSCRIPT - - # if Consul stops or fails, remove the config file before exiting the container - rm $CONSULSCRIPT - - echo '#' - echo '# Consul has stopped...' - echo '#' - ;; - - # - # A special, non-HA case for bootstrapping the HA raft - # - bootstrap) - set -eo pipefail - /bin/consul agent -config-dir=/config -server -bootstrap -ui-dir /ui - - # We only get here if Consul stops or fails... - echo '#' - echo '# Consul has stopped...' - echo '#' - ;; - - # - # The user wants to run something else... - # - *) - exec "$@" - ;; - -esac diff --git a/config/consul.json b/config/consul.json old mode 100755 new mode 100644 diff --git a/containerbuddy.json b/containerbuddy.json new file mode 100644 index 0000000..0b45d53 --- /dev/null +++ b/containerbuddy.json @@ -0,0 +1,12 @@ +{ + "consul": "127.0.0.1:8500", + "services": [ + { + "name": "consul", + "port": 8500, + "health": "/bin/consul-heartbeat", + "poll": 10, + "ttl": 25 + } + ] +} diff --git a/docker-compose.yml b/docker-compose.yml index 65414ac..4d63cd9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,32 +1,14 @@ -# -# Start a single consul host for bootstrapping the full consul cluster -# -consulbootstrap: - image: misterbisson/triton-consul:latest - command: bootstrap - restart: always - mem_limit: 128m - ports: - - 53 - - 8300 - - 8301 - - 8302 - - 8400 - - 8500 - dns: - - 127.0.0.1 - -# -# Start the following service with three nodes for reliability -# These instances depend on the bootstrap instance -# +# Start with a single host which will bootstrap the cluster. Once the +# cluster is bootstrapped we can set BOOTSTRAP_HOST and scale up +# more Consul instances. +# By using automatic bootstrapping via `-bootstrap-expect` and adding +# the environment variable, we can avoid having to use links. The +# environment variable is used once at bootstrap and then forever +# discarded by that node. consul: image: misterbisson/triton-consul:latest - command: raft restart: always mem_limit: 128m - links: - - consulbootstrap ports: - 53 - 8300 @@ -36,3 +18,12 @@ consul: - 8500 dns: - 127.0.0.1 + environment: + - BOOTSTRAP_HOST + command: > + /opt/containerbuddy/containerbuddy + -config file:///etc/containerbuddy.json + /bin/consul agent -server + -config-dir=/config + -ui-dir /ui + -bootstrap-expect 3 diff --git a/local-compose.yml b/local-compose.yml new file mode 100644 index 0000000..4ec13d7 --- /dev/null +++ b/local-compose.yml @@ -0,0 +1,5 @@ +consul: + extends: + file: docker-compose.yml + service: consul + build: . diff --git a/start.sh b/start.sh index 522f302..c39d95d 100755 --- a/start.sh +++ b/start.sh @@ -2,16 +2,23 @@ # check for prereqs command -v docker >/dev/null 2>&1 || { echo "Docker is required, but does not appear to be installed. See https://docs.joyent.com/public-cloud/api-access/docker"; exit; } -command -v sdc-listmachines >/dev/null 2>&1 || { echo "Joyent CloudAPI CLI is required, but does not appear to be installed. See https://apidocs.joyent.com/cloudapi/#getting-started"; exit; } -command -v json >/dev/null 2>&1 || { echo "JSON CLI tool is required, but does not appear to be installed. See https://apidocs.joyent.com/cloudapi/#getting-started"; exit; } -# manually name the project +# default values which can be overriden by -f or -p flags +export COMPOSE_FILE= export COMPOSE_PROJECT_NAME=consul +while getopts "f:p:" optchar; do + case "${optchar}" in + f) export COMPOSE_FILE=${OPTARG} ;; + p) export COMPOSE_PROJECT_NAME=${OPTARG} ;; + esac +done +shift $(expr $OPTIND - 1 ) + # give the docker remote api more time before timeout export DOCKER_CLIENT_TIMEOUT=300 -echo 'Starting a Triton trusted Compose service' +echo 'Starting a Triton trusted Consul service' echo echo 'Pulling the most recent images' @@ -19,60 +26,21 @@ docker-compose pull echo echo 'Starting containers' +export BOOTSTRAP_HOST= docker-compose up -d --no-recreate # Wait for the bootstrap instance echo echo -n 'Waiting for the bootstrap instance.' -export BOOTSTRAP_HOST="$(docker inspect -f '{{ .NetworkSettings.IPAddress }}' "${COMPOSE_PROJECT_NAME}_consulbootstrap_1"):8500" -ISRESPONSIVE=0 -while [ $ISRESPONSIVE != 1 ]; do - echo -n '.' - - curl -fs --connect-timeout 1 http://$BOOTSTRAP_HOST/ui &> /dev/null - if [ $? -ne 0 ] - then - sleep .3 - else - let ISRESPONSIVE=1 - fi -done -echo -echo 'The bootstrap instance is now running' -echo "Dashboard: $BOOTSTRAP_HOST/ui/" -command -v open >/dev/null 2>&1 && `open http://$BOOTSTRAP_HOST/ui/` - +export BOOTSTRAP_HOST="$(docker exec -it ${COMPOSE_PROJECT_NAME}_consul_1 ip addr show eth0 | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}')" +BOOTSTRAP_UI="$(docker inspect -f '{{ .NetworkSettings.IPAddress }}' "${COMPOSE_PROJECT_NAME}_consul_1")" -# Wait for, then bootstrap the first Consul raft instance -echo -echo -n 'Initilizing the Consul raft.' ISRESPONSIVE=0 while [ $ISRESPONSIVE != 1 ]; do echo -n '.' - RUNNING=$(docker inspect "${COMPOSE_PROJECT_NAME}_consul_1" | json -a State.Running) - if [ "$RUNNING" == "true" ] - then - docker exec -it "${COMPOSE_PROJECT_NAME}_consul_1" triton-bootstrap bootstrap - let ISRESPONSIVE=1 - else - sleep .3 - fi -done -echo - - - -# Wait for the first Consul raft instance -echo -echo -n 'Waiting for the first Consul raft instance to complete startup.' -export RAFT_HOST="$(docker inspect -f '{{ .NetworkSettings.IPAddress }}' "${COMPOSE_PROJECT_NAME}_consul_1"):8500" -ISRESPONSIVE=0 -while [ $ISRESPONSIVE != 1 ]; do - echo -n '.' - - curl -fs --connect-timeout 1 http://$RAFT_HOST/ui &> /dev/null + curl -fs --connect-timeout 1 http://$BOOTSTRAP_UI:8500/ui &> /dev/null if [ $? -ne 0 ] then sleep .3 @@ -81,11 +49,11 @@ while [ $ISRESPONSIVE != 1 ]; do fi done echo -echo 'The Consul raft is now running' -echo "Dashboard: $RAFT_HOST/ui/" -command -v open >/dev/null 2>&1 && `open http://$RAFT_HOST/ui/` +echo 'The bootstrap instance is now running' +echo "Dashboard: $BOOTSTRAP_UI:8500/ui/" +command -v open >/dev/null 2>&1 && `open http://$BOOTSTRAP_UI:8500/ui/` + -echo echo 'Scaling the Consul raft to three nodes' -echo "docker-compose -p=${COMPOSE_PROJECT_NAME} scale consul=3" -docker-compose -p=${COMPOSE_PROJECT_NAME} scale consul=3 +echo "docker-compose -p ${COMPOSE_PROJECT_NAME} scale consul=3" +docker-compose scale consul=3 diff --git a/test/lint.conf b/test/lint.conf new file mode 100644 index 0000000..c6ae00c --- /dev/null +++ b/test/lint.conf @@ -0,0 +1,127 @@ +# +# Configuration File for JavaScript Lint +# Developed by Matthias Miller (http://www.JavaScriptLint.com) +# +# This configuration file can be used to lint a collection of scripts, or to enable +# or disable warnings for scripts that are linted via the command line. +# + +### Warnings +# Enable or disable warnings based on requirements. +# Use "+WarningName" to display or "-WarningName" to suppress. +# ++ambiguous_else_stmt # the else statement could be matched with one of multiple if statements (use curly braces to indicate intent ++ambiguous_nested_stmt # block statements containing block statements should use curly braces to resolve ambiguity ++ambiguous_newline # unexpected end of line; it is ambiguous whether these lines are part of the same statement ++anon_no_return_value # anonymous function does not always return value ++assign_to_function_call # assignment to a function call +-block_without_braces # block statement without curly braces ++comma_separated_stmts # multiple statements separated by commas (use semicolons?) ++comparison_type_conv # comparisons against null, 0, true, false, or an empty string allowing implicit type conversion (use === or !==) ++default_not_at_end # the default case is not at the end of the switch statement ++dup_option_explicit # duplicate "option explicit" control comment ++duplicate_case_in_switch # duplicate case in switch statement ++duplicate_formal # duplicate formal argument {name} ++empty_statement # empty statement or extra semicolon ++identifier_hides_another # identifer {name} hides an identifier in a parent scope ++inc_dec_within_stmt # increment (++) and decrement (--) operators used as part of greater statement ++incorrect_version # Expected /*jsl:content-type*/ control comment. The script was parsed with the wrong version. ++invalid_fallthru # unexpected "fallthru" control comment ++invalid_pass # unexpected "pass" control comment ++jsl_cc_not_understood # couldn't understand control comment using /*jsl:keyword*/ syntax ++leading_decimal_point # leading decimal point may indicate a number or an object member ++legacy_cc_not_understood # couldn't understand control comment using /*@keyword@*/ syntax ++meaningless_block # meaningless block; curly braces have no impact ++mismatch_ctrl_comments # mismatched control comment; "ignore" and "end" control comments must have a one-to-one correspondence ++misplaced_regex # regular expressions should be preceded by a left parenthesis, assignment, colon, or comma ++missing_break # missing break statement ++missing_break_for_last_case # missing break statement for last case in switch ++missing_default_case # missing default case in switch statement ++missing_option_explicit # the "option explicit" control comment is missing ++missing_semicolon # missing semicolon ++missing_semicolon_for_lambda # missing semicolon for lambda assignment ++multiple_plus_minus # unknown order of operations for successive plus (e.g. x+++y) or minus (e.g. x---y) signs ++nested_comment # nested comment ++no_return_value # function {name} does not always return a value ++octal_number # leading zeros make an octal number ++parseint_missing_radix # parseInt missing radix parameter ++partial_option_explicit # the "option explicit" control comment, if used, must be in the first script tag ++redeclared_var # redeclaration of {name} ++trailing_comma_in_array # extra comma is not recommended in array initializers ++trailing_decimal_point # trailing decimal point may indicate a number or an object member ++undeclared_identifier # undeclared identifier: {name} ++unreachable_code # unreachable code ++unreferenced_argument # argument declared but never referenced: {name} ++unreferenced_function # function is declared but never referenced: {name} ++unreferenced_variable # variable is declared but never referenced: {name} ++unsupported_version # JavaScript {version} is not supported ++use_of_label # use of label ++useless_assign # useless assignment ++useless_comparison # useless comparison; comparing identical expressions ++useless_quotes # the quotation marks are unnecessary ++useless_void # use of the void type may be unnecessary (void is always undefined) ++var_hides_arg # variable {name} hides argument ++want_assign_or_call # expected an assignment or function call ++with_statement # with statement hides undeclared variables; use temporary variable instead + + +### Output format +# Customize the format of the error message. +# __FILE__ indicates current file path +# __FILENAME__ indicates current file name +# __LINE__ indicates current line +# __COL__ indicates current column +# __ERROR__ indicates error message (__ERROR_PREFIX__: __ERROR_MSG__) +# __ERROR_NAME__ indicates error name (used in configuration file) +# __ERROR_PREFIX__ indicates error prefix +# __ERROR_MSG__ indicates error message +# +# For machine-friendly output, the output format can be prefixed with +# "encode:". If specified, all items will be encoded with C-slashes. +# +# Visual Studio syntax (default): ++output-format __FILE__(__LINE__): __ERROR__ +# Alternative syntax: +#+output-format __FILE__:__LINE__: __ERROR__ + + +### Context +# Show the in-line position of the error. +# Use "+context" to display or "-context" to suppress. +# ++context + + +### Control Comments +# Both JavaScript Lint and the JScript interpreter confuse each other with the syntax for +# the /*@keyword@*/ control comments and JScript conditional comments. (The latter is +# enabled in JScript with @cc_on@). The /*jsl:keyword*/ syntax is preferred for this reason, +# although legacy control comments are enabled by default for backward compatibility. +# +-legacy_control_comments + + +### Defining identifiers +# By default, "option explicit" is enabled on a per-file basis. +# To enable this for all files, use "+always_use_option_explicit" ++always_use_option_explicit + +# Define certain identifiers of which the lint is not aware. +# (Use this in conjunction with the "undeclared identifier" warning.) +# +# Common uses for webpages might be: ++define require ++define console ++define process + +### JavaScript Version +# To change the default JavaScript version: +#+default-type text/javascript;version=1.5 +#+default-type text/javascript;e4x=1 + +### Files +# Specify which files to lint +# Use "+recurse" to enable recursion (disabled by default). +# To add a set of files, use "+process FileName", "+process Folder\Path\*.js", +# or "+process Folder\Path\*.htm". +# diff --git a/test/makefile b/test/makefile new file mode 100644 index 0000000..dc52333 --- /dev/null +++ b/test/makefile @@ -0,0 +1,11 @@ +PHONY: * + +build: + npm install . + +check: + jsstyle -o indent=4,unparenthesized-return=0,blank-after-open-comment=0 raft-test.js + jsl --verbose --nologo --nofilelisting --conf=lint.conf raft-test.js + +test: + node ./raft-test.js diff --git a/test/package.json b/test/package.json new file mode 100644 index 0000000..586be60 --- /dev/null +++ b/test/package.json @@ -0,0 +1,16 @@ +{ + "name": "consul-test", + "version": "0.1.0", + "description": "Perform behavioral tests on triton-consul raft", + "main": "raft-test.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "Joyent, Inc.", + "license": "Mozilla Public License 2.0", + "dependencies": { + "async": "^1.5.0", + "consul": "^0.18.1", + "dockerode": "^2.2.3" + } +} diff --git a/test/raft-test.js b/test/raft-test.js new file mode 100755 index 0000000..fe80586 --- /dev/null +++ b/test/raft-test.js @@ -0,0 +1,399 @@ +#!/usr/local/bin/node +var Docker = require('dockerode'); +var Consul = require('consul'); +var async = require('async'); + +// dockerode will automatically pick up your DOCKER_HOST, DOCKER_CERT_PATH +// but we need to set the version explicitly to support Triton +// ref https://github.com/apocas/dockerode/issues/154 +process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; +var docker = new Docker(); +docker.version = 'v1.20'; + +listConsul(function (err, consulNodes) { + if (err) { + console.log(err); + return; + } + switch (consulNodes.length) { + case 3: + console.log('Running 3-node raft tests.'); + run3NodeTests(consulNodes); + break; + case 5: + console.log('Running 5-node raft tests.'); + run5NodeTests(consulNodes); + break; + default: + console.log(consulNodes.length + ' Consul nodes up.'); + console.log('We need exactly 3 or 5 nodes to be up. Exiting.'); + break; + } + return; +}); + +// runs a series of tests on raft behavior for a 3-node Consul cluster +function run3NodeTests(consulNodes) { + console.log('Bootstrap node is:', consulNodes[0].Name); + waitForElection(consulNodes, function (err, result) { + if (err) { + console.log(err); + return; + } + console.log('raft is healthy: ', result); + async.series([ + function (cb) { test3_1(consulNodes, cb); }, + function (cb) { test3_2(consulNodes, cb); }, + function (cb) { test3_3(consulNodes, cb); } + ], function (errResult, testResults) { + if (errResult) { + console.log(errResult); + } else { + for (var i in testResults) { + if (!testResults[i]) { + console.log('Failed!', testResults); + return; + } + } + console.log('Passed!'); + } + }); + }); +} + +// Runs a series of tests on raft behavior for a 5-node Consul cluster +function run5NodeTests(consulNodes) { + console.log('Bootstrap node is:', consulNodes[0].Names[0]); + waitForElection(consulNodes, function (err, result) { + if (err) { + console.log(err); + return; + } + console.log('raft is healthy: ', result); + async.series([ + function (cb) { test5_1(consulNodes, cb); }, + function (cb) { test5_2(consulNodes, cb); } + ], function (errResult, testResults) { + if (errResult) { + console.log(errResult); + } else { + for (var i in testResults) { + if (!testResults[i]) { + console.log('Failed!', testResults); + return; + } + } + console.log('Passed!'); + } + }); + }); +} + + +// Test that a non-bootstrap node rejoins the raft after reboot +function test3_1(consulNodes, callback) { + console.log('[test3_1] -----------------------------'); + var consul1 = consulNodes[0], + consul2 = consulNodes[1], + consul3 = consulNodes[2]; + + async.series([ + function (cb) { stop(consul3, cb); }, + function (cb) { waitForElection([consul1, consul2], cb); }, + function (cb) { start(consul3, cb); }, + function (cb) { waitForElection(consulNodes, cb); } + ], function (err, results) { + callback(err, results[results.length - 1]); + }); +} + +// Test that the bootstrap node rejoins the same raft after reboot +function test3_2(consulNodes, callback) { + console.log('[test3_2] -----------------------------'); + var consul1 = consulNodes[0], + consul2 = consulNodes[1], + consul3 = consulNodes[2]; + + async.series([ + function (cb) { stop(consul1, cb); }, + function (cb) { waitForElection([consul2, consul3], cb); }, + function (cb) { start(consul1, cb); }, + function (cb) { waitForElection(consulNodes, cb); } + ], function (err, results) { + callback(err, results[results.length - 1]); + }); +} + +// Test that non-bootstrap nodes rejoin the raft even if bootstrap +// node is gone +function test3_3(consulNodes, callback) { + console.log('[test3_3] -----------------------------'); + var consul1 = consulNodes[0], + consul2 = consulNodes[1], + consul3 = consulNodes[2]; + + async.series([ + function (cb) { stop(consul1, cb); }, + function (cb) { stop(consul3, cb); }, + function (cb) { start(consul3, cb); }, + function (cb) { waitForElection([consul3, consul2], cb); }, + function (cb) { start(consul1, cb); }, + function (cb) { waitForElection(consulNodes, cb); } + ], function (err, results) { + callback(err, results[results.length - 1]); + }); +} + +// Test that consistent reads fail without quorum but that they become +// available after partition heals +function test5_1(consulNodes, callback) { + var consul1 = consulNodes[0], + consul2 = consulNodes[1], + consul3 = consulNodes[2], + consul4 = consulNodes[3], + consul5 = consulNodes[4]; + + async.series([ + function (cb) { stop(consul1, cb); }, + function (cb) { stop(consul2, cb); }, + function (cb) { testWrites([consul3, consul4, consul5], + 'stale', true, cb); }, + function (cb) { testWrites([consul3, consul4, consul5], + 'consistent', false, cb); }, + function (cb) { start(consul2, cb); }, + function (cb) { waitForElection(consulNodes.slice(1), cb); }, + function (cb) { testWrites([consul3, consul4, consul5], + 'consistent', true, cb); }, + function (cb) { start(consul1, cb); }, + function (cb) { waitForElection(consulNodes, cb); } + ], function (err, results) { + callback(err, results[results.length - 1]); + }); +} + +// Test that majority writes win after raft heals +function test5_2(consulNodes, callback) { + var consul1 = consulNodes[0], + consul2 = consulNodes[1], + consul3 = consulNodes[2], + consul4 = consulNodes[3], + consul5 = consulNodes[4]; + + async.series([ + function (cb) { createNetsplit([consul1, consul2], + [consul3, consul4, consul5], cb); }, + function (cb) { testWrites([consul1, consul2], + 'stale', true, cb); }, + function (cb) { testWrites([consul1, consul2], + 'consistent', false, cb); }, + function (cb) { healNetsplit([consul1, consul2, consul3, + consul4, consul5], cb); }, + function (cb) { waitForElection(consulNodes, cb); }, + function (cb) { testWrites([consul1, consul2], + 'consistent', true, cb); } + ], function (err, results) { + callback(err, results[results.length - 1]); + }); +} + + +// Queries Consul to determine the status of the raft. Compares the status +// against a list of containers and verifies that the leader is among those +// nodes. If failing, will retry twice with some backoff and then return +// error to the callback if the raft still has not healed. +// @param {containers} array of container objects from our Consul nodes +// array that should be members of the raft. +// @callback {callback} function(err, result) +function waitForElection(containers, callback) { + + var expected = []; + containers.forEach(function (container) { + expected.push(container.Ip+':8300'); + }); + expected.sort(); + console.log('expected peers:', expected.toString()); + + var isMatch = false; + var count = 0; + var maxCount = 3; + + async.doUntil( + function (cb) { + getLeader(containers[0], function (err, leader) { + if (err || !leader) { + cb(err); + return; + } + isMatch = (expected.indexOf(leader) != -1); + cb(null); + }); + }, + function () { + count++; + return (isMatch || count >= maxCount); + }, + function (err) { + if (err) { + return callback(err, false); + } else { + if (!isMatch) { + return callback(('error: raft leader is not '+ + 'among expected nodes'), null); + } + console.log('raft leader is among expected nodes'); + return callback(null, true); + } + }); +} + +// @param {[containers]} array of container objects from our Consul +// nodes array that we want to test writes against +// @callback {callback} function(err, result) +function testWrites(containers, consistency, expectPass, callback) { + // TODO: implementation + console.log('testWrites:', containers.length, consistency, + 'expectPass:', expectPass); + callback(null, 'testWrites'); +} + +function createNetsplit(group1, group2, callback) { + // TODO: implementation + console.log(group1); + console.log(group2); + console.log(callback); +} + +function healNetsplit(containers, callback) { + // TODO: implementation + console.log(containers); + console.log(callback); +} + + +// Create an array of containers labelled with the service 'consul', +// sorted by name +// @callback {fn} function(err, result) +function listConsul(callback) { + var consul = []; + docker.listContainers( + {all: false, + filters: ['{"label":["com.docker.compose.service=consul"]}']}, + function (err, containers) { + if (err) { + callback(err, null); + return; + } + async.each(containers, function (container, cb) { + var cmd = ['ip', 'addr', 'show', 'eth0']; + runExec(container, cmd, function (e, ip) { + if (e) { + cb(e); + return; + } + container['Ip'] = matchIp(ip); + cb(null); + }); + }, function (inspectErr) { + if (inspectErr) { + callback(inspectErr, null); + } + containers.forEach(function (container) { + container['Name'] = container.Names[0].replace('/', ''); + consul.push(container); + }); + consul.sort(byName); + callback(null, consul); + return; + }); + }); +} + +function byName(a, b) { + var x = a.Names[0]; var y = b.Names[0]; + return ((x < y) ? -1 : ((x > y) ? 1 : 0)); +} + +// @callback {fn} function(err, result) +function stop(container, fn) { + console.log('stopping', container.Name); + docker.getContainer(container.Id).stop(fn); +} + +// @callback {fn} function(err, result) +function start(container, fn) { + console.log('starting', container.Name); + docker.getContainer(container.Id).start(fn); +} + +// @callback {fn} function(err, leader) +function getLeader(container, fn) { + runExec(container, + ['curl', '127.0.0.1:8500/v1/status/leader'], + function (err, leader) { + if (err || leader === null) { + return fn(err, null); + } + return fn(null, matchIpPort(leader)[0]); + }); +} + +// @callback {fn} function(err, peers) +// peers will be an array of strings in the form "{ip}:{port}" +function getPeers(container, fn) { + runExec(container, + ['curl', '-s', '127.0.0.1:8500/v1/status/peers'], + function (err, peers) { + if (err || peers === null) { + return fn(err, null); + } + return fn(null, matchIpPort(peers)); + }); +} + +// returns a string +function matchIp(input) { + return input.match(/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/); +} + +// returns an array of strings in the form ["{ip}:{port}"] +function matchIpPort(input) { + return input.match(/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}:8300/g); +} + +// Runs `docker exec` and concatenates stream into a single `results` +// string for the callback. +// @param {container} container to run command on +// @param {command} command and args ex. ['curl', '-v', 'example.com'] +// @callback {callback(err, results)} results +function runExec(container, command, callback) { + var options = { + AttachStdin: true, + AttachStdout: true, + AttachStderr: true, + Tty: true, + Cmd: command + }; + + docker.getContainer(container.Id).exec(options, function (execErr, exec) { + if (execErr) { + callback(execErr, null); + } + exec.start({Tty: true, Detach: true}, function (err, stream) { + if (err) { + callback(err, null); + } + var body = ''; + stream.setEncoding('utf8'); + stream.once('error', function (error) { + callback(error, null); + }); + stream.once('end', function () { + callback(null, body); + }); + stream.on('data', function (chunk) { + body += chunk; + }); + + }); + }); +}