From 627039e06b7010130b02047815552c0eadfffa53 Mon Sep 17 00:00:00 2001 From: Clinton Kitson Date: Sat, 6 Feb 2016 20:26:42 -0800 Subject: [PATCH] Added dynamic config and external volume capabilities This commit introduces changes to the configuration to include the ability to customize any number of nodes and personalities. It also includes external volume support. --- .gitignore | 1 + README.md | 149 ++++++++++++++++++++++++- Vagrantfile | 223 ++++++++++++++++++++++++++++--------- bin/test | 2 +- config.json | 16 ++- lib/ruby/playa_settings.rb | 2 +- 6 files changed, 326 insertions(+), 67 deletions(-) diff --git a/.gitignore b/.gitignore index e06f989..61c70b8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .DS_Store packer_cache packer/builds +Volumes/* diff --git a/README.md b/README.md index c3379a5..127b48b 100644 --- a/README.md +++ b/README.md @@ -3,30 +3,131 @@ [Playa Mesos][8] helps you quickly create [Apache Mesos][1] test environments. This project relies on [VirtualBox][5], [Vagrant][6], and an Ubuntu box image which has Mesos and [Marathon][2] pre-installed. The box image is downloadable for your -convenience, but it can also be built from source using [Packer][9]. +convenience, but it can also be built from source using [Packer][9]. There is an external volume capability when using +with VirtualBox provided through [REX-Ray](https://github.com/emccode/rexray). As an alternative to VirtualBox, it's possible to build and run the image on VMware [Fusion](https://www.vmware.com/products/fusion/) or [Workstation](https://www.vmware.com/products/workstation/). ## Requirements -* [VirtualBox][5] 4.2+ +* [VirtualBox][5] 5.0.10+ * [Vagrant][6] 1.3+ * [git](http://git-scm.com/downloads) (command line tool) * [Packer][9] 0.5+ (optional) * VMware [Fusion](https://www.vmware.com/products/fusion/) or [Workstation](https://www.vmware.com/products/workstation/) (optional) * [Vagrant Plugin for VMware Fusion or Workstation](https://www.vagrantup.com/vmware) (optional) +## Configuration +The `config.json` file holds all of the configurable parameters. By default +there is one host deployed running the master and slave services. Additionally +both Chronos and Marathon frameworks are running. Set the appropriate hostnames, IP +addresses, VM settings, and other parameters in this file. + +### Example All-In-One Master +```json +{ + "platform": "virtualbox", + "box_name": "playa_mesos_ubuntu_14.04_201601041324", + "base_url": "http://downloads.mesosphere.io/playa-mesos", + "hosts": { + "mesos-master": { + "ip": "10.141.141.10", + "vm_ram": "2048", + "vm_cpus": "2" + } + } +} +``` + +### Example All-In-One Master with External Volumes +The `external_volumes` parameter can be introduced which leverages VirtualBox +to provide external volume support for tasks. The external volume functionality +means that both the `Mesos containerizer` and `Docker containerizer` can +define external volumes that get attached to tasks. See [Mesos](https://github.com/emccode/mesos-module-dvdi) +and [Docker](https://github.com/emccode/rexray) for examples. + + + +```json +{ + "platform": "virtualbox", + "box_name": "playa_mesos_ubuntu_14.04_201601041324", + "base_url": "http://downloads.mesosphere.io/playa-mesos", + "hosts": { + "mesos-master": { + "ip": "10.141.141.10", + "vm_ram": "2048", + "vm_cpus": "2", + "external_volumes":true + } + } +} +``` + +### Example Master with 3 Slaves and External Volumes +Optionally a more realistic set of nodes can be deployed. We include an +example below for `mesos-master`, `mesos-slave1`, `mesos-slave2`, and +`mesos-slave3`. There is a `disable_slave` flag that can be set per node to +determine the personality of the node which is used on `mesos-master`. +The root `vm_ram` and `vm_cpus` will be applied where configuration is not set +on individual nodes. + +```json +{ + "platform": "virtualbox", + "box_name": "playa_mesos_ubuntu_14.04_201601041324", + "base_url": "http://downloads.mesosphere.io/playa-mesos", + "hosts": { + "mesos-master": { + "ip": "10.141.141.10", + "vm_ram": "512", + "vm_cpus": "1", + "disable_slave": true + }, + "mesos-slave1":{ + "ip":"10.141.141.11", + "external_volumes":true + }, + "mesos-slave2":{ + "ip":"10.141.141.12", + "external_volumes":true + }, + "mesos-slave3":{ + "ip":"10.141.141.13", + "external_volumes":true + } + }, + "vm_ram": "1024", + "vm_cpus": "1" +} +``` + +## Runtime Note +At any time you can remove a host or add a new host with new +settings. + +```bash +vagrant destroy -f mesos-slave1 && vagrant up mesos-slave1 +``` + ## Quick Start 1. [Install VirtualBox](https://www.virtualbox.org/wiki/Downloads) 1. [Install Vagrant](http://www.vagrantup.com/downloads.html) +1. (Optional for external volumes) Disable authentication for VirtualBox and start the SOAP Web Service. + + ```bash + VBoxManage setproperty websrvauthlibrary null + /Applications/VirtualBox.app/Contents/MacOS/vboxwebsrv -H 0.0.0.0 -v + ``` + 1. Clone this repository ```bash - git clone https://github.com/mesosphere/playa-mesos.git + git clone https://github.com/mesosphere/playa-mesos cd playa-mesos ``` @@ -47,7 +148,7 @@ VMware [Fusion](https://www.vmware.com/products/fusion/) or [Workstation](https: 1. SSH to the VM ```bash - vagrant ssh + vagrant ssh mesos-master ps -eaf | grep mesos exit ``` @@ -104,6 +205,42 @@ The build is controlled with the following files: For additional information on customizing the build, or creating a new profile, see [Configuration][15] and the [Packer Documentation][20]. +## External Volume Examples +There are two personalities being advertised from the external volume driver. +The first is for native Docker volumes through the `docker` Volume Driver name. +This is where a Docker containerizer is being used. + +```bash +docker run -ti --volume-driver=docker -v test:/test busybox +df /test +exit +``` + +The second is through the Mesos containerizer through the `mesos` Volume Driver +name. See the following Marathon example `job.json` file. + +```json +{ + "id": "hello-play", + "cmd": "while [ true ] ; do touch /tmp/hello ; sleep 5 ; done", + "mem": 32, + "cpus": 0.1, + "instances": 1, + "env": { + "DVDI_VOLUME_NAME": "test2", + "DVDI_VOLUME_DRIVER": "mesos", + "DVDI_VOLUME_OPTS": "size=5" + } +} +``` + +Start the job interactively through Marathon. + +```bash +vagrant ssh mesos-master +http post http://127.0.0.1:8080/v2/apps < job.json +``` + ## Documentation * [Configuration][15] @@ -116,7 +253,7 @@ see [Configuration][15] and the [Packer Documentation][20]. * [vagrant-mesos](https://github.com/everpeace/vagrant-mesos): Vagrant provisioning with multinode and EC2 support -* [babushka-mesos](https://github.com/parolkar/mesos-babushka): It is [Babushka](http://babushka.me/) based provisioning of Mesos Cluster which can help you demonstrate [potential](http://vimeo.com/110914075) of mesos. +* [babushka-mesos](https://github.com/parolkar/mesos-babushka): It is [Babushka](http://babushka.me/) based provisioning of Mesos Cluster which can help you demonstrate [potential](http://vimeo.com/110914075) of mesos. ## Authors @@ -146,3 +283,5 @@ VMware Support: [Fabio Rapposelli](https://github.com/frapposelli) ([@fabiorappo [21]: config.json "config.json" [22]: packer/packer.json "packer.json" [23]: lib/scripts "scripts" +[24]: https://github.com/emccode/rexray "REX-Ray" +[25]: https://github.com/emccode/mesos-module-dvdi "mesos-module-dvdi" diff --git a/Vagrantfile b/Vagrantfile index 24708f2..f55f757 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -19,65 +19,180 @@ ENV['VAGRANT_DEFAULT_PROVIDER'] = pmconf.platform Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - # Create a private network, which allows host-only access to the machine - # using a specific IP. - config.vm.network :private_network, ip: pmconf.ip_address - - # If true, then any SSH connections made will enable agent forwarding. - # Default value: false - config.ssh.forward_agent = true - - # Every Vagrant virtual environment requires a box to build off of. - config.vm.box = pmconf.box_name - - # There are two levels of caching here. - # 1. If pmconf.box_local is set, then the file referenced by pmconf.box_url - # was found in the Packer build path (packer/builds/*.box) and Vagrant's - # vm.box_url is set to that path. To force retrieving the box - # from the URL again, simply remove the Packer builds directory. - # 2. Vagrant only retrieves box images from vm.box_url if it does not have - # a local copy in ~/.vagrant.d/boxes/$BOX_NAME. These can be removed - # with the command: "vagrant box remove $BOX_NAME" - config.vm.box_url = pmconf.box_local ? pmconf.box_local : box_url - - # Note: You'll want a decent amount of memory for your mesos master/slave - # VM. The strict minimum, at least while the VM is provisioned, is the - # amount necessary to compile mesos and the jenkins plugin. 2048m+ is - # recommended. The CPU count can be lowered, but you may run into issues - # running the Jenkins Mesos Framework if you do so. - config.vm.provider :virtualbox do |vb| - vb.name = pmconf.box_name - vb.customize ['modifyvm', :id, '--memory', pmconf.vm_ram] - vb.customize ['modifyvm', :id, '--cpus', pmconf.vm_cpus] - end - config.vm.provider :vmware_fusion do |v| - v.vmx['memsize'] = pmconf.vm_ram - v.vmx['numvcpus'] = pmconf.vm_cpus - end - config.vm.provider :vmware_workstation do |v| - v.vmx['memsize'] = pmconf.vm_ram - v.vmx['numvcpus'] = pmconf.vm_cpus - end + hosts = pmconf.hosts + + hosts.each_key do |hostname| + config.vm.define hostname do |node| + + node.vm.hostname = hostname + + hostip = hosts[hostname]["ip"] + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + node.vm.network :private_network, ip: hostip + + # If true, then any SSH connections made will enable agent forwarding. + # Default value: false + node.ssh.forward_agent = true + + # Every Vagrant virtual environment requires a box to build off of. + node.vm.box = pmconf.box_name + + # There are two levels of caching here. + # 1. If pmconf.box_local is set, then the file referenced by pmconf.box_url + # was found in the Packer build path (packer/builds/*.box) and Vagrant's + # vm.box_url is set to that path. To force retrieving the box + # from the URL again, simply remove the Packer builds directory. + # 2. Vagrant only retrieves box images from vm.box_url if it does not have + # a local copy in ~/.vagrant.d/boxes/$BOX_NAME. These can be removed + # with the command: "vagrant box remove $BOX_NAME" + node.vm.box_url = pmconf.box_local ? pmconf.box_local : box_url + + # Note: You'll want a decent amount of memory for your mesos master/slave + # VM. The strict minimum, at least while the VM is provisioned, is the + # amount necessary to compile mesos and the jenkins plugin. 2048m+ is + # recommended. The CPU count can be lowered, but you may run into issues + # running the Jenkins Mesos Framework if you do so. + node.vm.provider :virtualbox do |vb| + vb.name = hostname + vb.customize ['modifyvm', :id, '--memory', hosts[hostname]["vm_ram"] ? hosts[hostname]["vm_ram"] : pmconf.vm_ram] + vb.customize ['modifyvm', :id, '--cpus', hosts[hostname]["vm_cpus"] ? hosts[hostname]["vm_cpus"] : pmconf.vm_cpus] + vb.customize ["storagectl", :id, "--add", "sata", "--controller", "IntelAhci", "--name", "SATA", "--portcount", 30, "--hostiocache", "on"] + vb.customize ["modifyvm", :id, "--macaddress1", "auto"] + end + node.vm.provider :vmware_fusion do |v| + v.vmx['memsize'] = pmconf.vm_ram + v.vmx['numvcpus'] = pmconf.vm_cpus + end + node.vm.provider :vmware_workstation do |v| + v.vmx['memsize'] = pmconf.vm_ram + v.vmx['numvcpus'] = pmconf.vm_cpus + end + + # Make the project root available to the guest VM. + # node.vm.synced_folder '.', '/vagrant' + + dir = "#{ENV['PWD']}/Volumes" + + master_ip = hosts["mesos-master"]["ip"] + + # Only provision if explicitly request with 'provision' or 'up --provision' + if ARGV.any? { |arg| arg =~ /^(--)?provision$/ } + node.vm.provision :shell do |shell| + shell.path = 'lib/scripts/common/mesosflexinstall' + arg_array = ['--slave-hostname', hostip] - # Make the project root available to the guest VM. - # config.vm.synced_folder '.', '/vagrant' + # If mesos_release exists in the node.json file, pass the '--rel' + # argument and a version to mesosflexinstall. Otherwise, do nothing. + if pmconf.instance_variable_get(:@settings).include?('mesos_release') + arg_array += ['--rel', pmconf.mesos_release] + end - # Only provision if explicitly request with 'provision' or 'up --provision' - if ARGV.any? { |arg| arg =~ /^(--)?provision$/ } - config.vm.provision :shell do |shell| - shell.path = 'lib/scripts/common/mesosflexinstall' - arg_array = ['--slave-hostname', pmconf.ip_address] + # Using an array for shell args requires Vagrant 1.4.0+ + # TODO: Set as array directly when Vagrant 1.3 support is dropped + shell.args = arg_array.join(' ') + end + end + + if hostname == "mesos-master" + node.vm.provision "shell", inline: <<-SHELL + echo #{master_ip} > /etc/mesos-master/advertise_ip + service mesos-master restart + apt-get install -y httpie + SHELL + end + + if hosts[hostname]["disable_slave"] == true + node.vm.provision "shell", inline: <<-SHELL + service mesos-slave stop + echo manual > /etc/init/mesos-slave.override + SHELL + end - # If mesos_release exists in the config.json file, pass the '--rel' - # argument and a version to mesosflexinstall. Otherwise, do nothing. - if pmconf.instance_variable_get(:@settings).include?('mesos_release') - arg_array += ['--rel', pmconf.mesos_release] + if hostname != "mesos-master" + node.vm.provision "shell", inline: <<-SHELL + service marathon stop + echo manual > /etc/init/marathon.override + service chronos stop + echo manual > /etc/init/chronos.override + service mesos-master stop + echo manual > /etc/init/zookeeper.override + service zookeeper stop + echo manual > /etc/init/mesos-master.override + service mesos-slave stop + + rm -Rf /tmp/mesos/meta + + echo #{hostip} > /etc/mesos-slave/hostname + echo #{hostip} > /etc/mesos-slave/ip + echo MASTER=zk://#{master_ip}:2181/mesos > /etc/default/mesos-slave + service mesos-slave start + SHELL + end + + if hosts[hostname]["external_volumes"] == true + node.vm.provision "shell", inline: <<-SHELL + sudo curl -sSL https://dl.bintray.com/emccode/rexray/install | sh -s staged + + sudo tee -a /etc/rexray/config.yml << EOF +rexray: + storageDrivers: + - virtualbox + modules: + default-docker: + host: "unix:///run/docker/plugins/docker.sock" + rexray: + volume: + mount: + preempt: true + mesos: + type: docker + host: "unix:///run/docker/plugins/mesos.sock" + rexray: + volume: + mount: + preempt: true + ignoreUsedCount: true +virtualbox: + endpoint: http://10.0.2.2:18083 + tls: false + volumePath: /Users/user/VirtualBox Volumes + controllerName: SATA +EOF + + sed -i '/.*volumePath.*/c\\\x20\x20volumePath: \"#{dir}\"' /etc/rexray/config.yml + + wget -nv --directory-prefix=/usr/lib https://github.com/emccode/mesos-module-dvdi/releases/download/v0.4.0/libmesos_dvdi_isolator-0.26.0.so 1&>2 + sudo tee -a /usr/lib/dvdi-mod.json << EOF +{ + "libraries": [ + { + "file": "/usr/lib/libmesos_dvdi_isolator-0.26.0.so", + "modules": [ + { + "name": "com_emccode_mesos_DockerVolumeDriverIsolator" + } + ] + } + ] +} +EOF + + echo file:///usr/lib/dvdi-mod.json > /etc/mesos-slave/modules + echo posix/cpu,posix/mem,com_emccode_mesos_DockerVolumeDriverIsolator > /etc/mesos-slave/isolation + curl -sSL https://dl.bintray.com/emccode/dvdcli/install | sh - + + sudo rexray start + SHELL + end + + if !hosts[hostname].has_key?("disable_slave") || hosts[hostname]["disable_slave"] == false + node.vm.provision "shell", inline: <<-SHELL + service mesos-slave restart + SHELL end - # Using an array for shell args requires Vagrant 1.4.0+ - # TODO: Set as array directly when Vagrant 1.3 support is dropped - shell.args = arg_array.join(' ') end end - end diff --git a/bin/test b/bin/test index 4c952c9..97f1174 100755 --- a/bin/test +++ b/bin/test @@ -7,7 +7,7 @@ declare -r pm_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" . "${pm_root}/lib/assert.sh/assert.sh" # Requirements -declare -r min_vbox_ver=4.2 +declare -r min_vbox_ver=5.0.10 declare -r min_vagrant_ver=1.3 # Ensure required commands are available diff --git a/config.json b/config.json index dd0e0d9..9707f4a 100644 --- a/config.json +++ b/config.json @@ -1,8 +1,12 @@ { - "platform": "virtualbox", - "box_name": "playa_mesos_ubuntu_14.04_201601041324", - "base_url": "http://downloads.mesosphere.io/playa-mesos", - "ip_address": "10.141.141.10", - "vm_ram": "2048", - "vm_cpus": "2" + "platform": "virtualbox", + "box_name": "playa_mesos_ubuntu_14.04_201601041324", + "base_url": "http://downloads.mesosphere.io/playa-mesos", + "hosts": { + "mesos-master": { + "ip": "10.141.141.10", + "vm_ram": "2048", + "vm_cpus": "2" + } + } } diff --git a/lib/ruby/playa_settings.rb b/lib/ruby/playa_settings.rb index 644bf54..7d9e096 100644 --- a/lib/ruby/playa_settings.rb +++ b/lib/ruby/playa_settings.rb @@ -14,7 +14,7 @@ def method_missing(sym, *args, &block) if @settings.include?(sym.to_s) return @settings[sym.to_s] else - super(sym, *args, &block) + #super(sym, *args, &block) end end