Merge pull request #191 from cloudamatic/development

Spring Cleanse

Dockerfile

@@ -8,7 +8,7 @@ RUN df -h
 
 RUN apt-get update
 
-RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential
+RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential python-pip curl
 
 RUN apt-get upgrade -y
 
@@ -24,10 +24,14 @@ RUN ls -la
 
 #RUN rm --verbose -f cloud-mu-*.gem
 
+RUN pip install pywinrm
+
 RUN apt-get remove -y build-essential ruby2.5-dev
 
 RUN apt-get autoremove -y
 
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.17.4/bin/linux/amd64/kubectl && mv kubectl /usr/bin && chmod +x /usr/bin/kubectl
+
 EXPOSE 2260
 
 CMD /usr/sbin/init

ansible/roles/mu-windows/files/LaunchConfig.json

@@ -0,0 +1,9 @@
+{
+  "setComputerName": false,
+  "setMonitorAlwaysOn": true,
+  "setWallpaper": true,
+  "addDnsSuffixList": true,
+  "extendBootVolumeSize": true,
+  "handleUserData": true,
+  "adminPasswordType": "Random"
+}

ansible/roles/mu-windows/files/config.xml

@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Ec2ConfigurationSettings>
+  <Plugins>
+    <Plugin>
+      <Name>Ec2SetPassword</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2SetComputerName</Name>
+      <State>Disabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2InitializeDrives</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2EventLog</Name>
+      <State>Disabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2ConfigureRDP</Name>
+      <State>Disabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2OutputRDPCert</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2SetDriveLetter</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2WindowsActivate</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2DynamicBootVolumeSize</Name>
+      <State>Disabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2SetHibernation</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2SetMonitorAlwaysOn</Name>
+      <State>Disabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2ElasticGpuSetup</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2FeatureLogging</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2SetENAConfig</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>Ec2HandleUserData</Name>
+      <State>Enabled</State>
+    </Plugin>
+    <Plugin>
+      <Name>AWS.EC2.Windows.CloudWatch.PlugIn</Name>
+      <State>Disabled</State>
+    </Plugin>
+  </Plugins>
+  <GlobalSettings>
+    <ManageShutdown>true</ManageShutdown>
+    <SetDnsSuffixList>true</SetDnsSuffixList>
+    <WaitForMetaDataAvailable>true</WaitForMetaDataAvailable>
+    <ShouldAddRoutes>true</ShouldAddRoutes>
+    <RemoveCredentialsfromSysprepOnStartup>true</RemoveCredentialsfromSysprepOnStartup>
+  </GlobalSettings>
+</Ec2ConfigurationSettings>

ansible/roles/mu-windows/tasks/main.yml

@@ -18,3 +18,19 @@
   win_chocolatey:
     name: openssh
     state: present
+
+- name: "Tell EC2Config to set a random password on next boot (Windows 2012)"
+  when: ((ansible_facts['distribution_major_version'] | int) < 10 and mu_build_image is defined and mu_build_image == True)
+  win_copy:
+    src: config.xml
+    dest: "c:/Program Files/Amazon/EC2ConfigService/Settings/config.xml"
+
+- name: "Tell EC2Launch to set a random password (Windows 2016+)"
+  when: ((ansible_facts['distribution_major_version'] | int) >= 10 and mu_build_image is defined and mu_build_image == True)
+  win_copy:
+    src: LaunchConfig.json
+    dest: "c:/ProgramData/Amazon/EC2-Windows/Launch/Config/LaunchConfig.json"
+
+- name: "Tell EC2Launch to run on next boot (Windows 2016+)"
+  when: ((ansible_facts['distribution_major_version'] | int) >= 10 and mu_build_image is defined and mu_build_image == True)
+  win_shell: C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 -Schedule

bin/mu-adopt

@@ -48,6 +48,7 @@ $opt = Optimist::options do
   opt :diff, "List the differences between what we find and an existing, saved deploy from a previous run, if one exists.", :required => false, :type => :boolean
   opt :grouping, "Methods for grouping found resources into separate Baskets.\n\n"+MU::Adoption::GROUPMODES.keys.map { |g| "* "+g.to_s+": "+MU::Adoption::GROUPMODES[g] }.join("\n")+"\n\n", :required => false, :type => :string, :default => "logical"
   opt :habitats, "Limit scope of searches to the named accounts/projects/subscriptions, instead of search all habitats visible to our credentials.", :required => false, :type => :strings
+  opt :scrub, "Whether to set scrub_mu_isms in the BoKs we generate", :default => $MU_CFG.has_key?('adopt_scrub_mu_isms') ? $MU_CFG['adopt_scrub_mu_isms'] : false
 end
 
 ok = true
@@ -102,7 +103,7 @@ if !ok
 end
 
 
-adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats])
+adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub])
 found = adoption.scrapeClouds
 if found.nil? or found.empty?
   MU.log "No resources found to adopt", MU::WARN, details: {"clouds" => clouds, "types" => types }

bin/mu-configure

@@ -113,6 +113,12 @@ $CONFIGURABLES = {
     "desc" => "Disable the Momma Cat grooming daemon. Nodes which require asynchronous Ansible/Chef bootstraps will not function. This option is only honored in gem-based installations.",
     "boolean" => true
   },
+  "adopt_scrub_mu_isms" => {
+    "title" => "Disable Momma Cat",
+    "default" => false,
+    "desc" => "Ordinarily, Mu will automatically name, tag and generate auxiliary resources in a standard Mu-ish fashion that allows for deployment of multiple clones of a given stack. Toggling this flag will change the default behavior of mu-adopt, when it creates stack descriptors from found resources, to enable or disable this behavior (see also mu-adopt's --scrub option).",
+    "boolean" => true
+  },
   "mommacat_port" => {
     "title" => "Momma Cat Listen Port",
     "pattern" => /^[0-9]+$/i,
@@ -246,6 +252,16 @@ $CONFIGURABLES = {
         "required" => false,
         "desc" => "For Google Cloud projects which are attached to a GSuite domain. Some API calls (groups, users, etc) require this identifier. From admin.google.com, choose Security, the Single Sign On, and look for the Entity ID field. The value after idpid= in the URL there should be the customer ID."
       },
+      "ignore_habitats" => {
+        "title" => "Ignore These Projects",
+        "desc" => "Optional list of projects to ignore, for credentials which have visibility into multiple projects",
+        "array" => true
+      },
+      "restrict_to_habitats" => {
+        "title" => "Operate On Only These Projects",
+        "desc" => "Optional list of projects to which we'll restrict all of our activities.",
+        "array" => true
+      },
       "default" => {
         "title" => "Is Default Account",
         "default" => false,

bin/mu-node-manage

@@ -29,18 +29,20 @@ Usage:
   opt :all, "Operate on all nodes/deploys. Use with caution.", :require => false, :default => false, :type => :boolean
   opt :platform, "Operate exclusively on one nodes of a particular operating system. Can be used in conjunction with -a or -d. Valid platforms: linux, windows", :require => false, :type => :string
   opt :environment, "Operate exclusively on one nodes with a particular environment (e.g. dev, prod). Can be used in conjunction with -a or -d.", :require => false, :type => :string
-  opt :override_chef_runlist, "An alternate runlist to pass to Chef, in chefrun mode.", :require => false, :type => :string
+  opt :override_chef_runlist, "An alternate runlist to pass to Chef, in groomeronly mode.", :require => false, :type => :string
   opt :xecute, "Run a shell command on matching nodes. Overrides --mode and suppresses some informational output in favor of scriptability.", :require => false, :type => :string
-  opt :mode, "Action to perform on matching nodes. Valid actions: groom, chefrun, awsmeta, vaults, certs, chefupgrade", :require => false, :default => "chefrun", :type => :string
+  opt :mode, "Action to perform on matching nodes. Valid actions: groom, groomeronly, awsmeta, vaults, certs, chefupgrade", :require => false, :default => "groomeronly", :type => :string
   opt :verbose, "Show output from Chef runs, etc", :require => false, :default => false, :type => :boolean
   opt :winrm, "Force WinRM connection. Disable SSH fallback", :require => false, :default => false, :type => :boolean
   opt :info, "List a particular node attribute", :require => false, :default => 'nodename', :type => :string
 end
 
 MU.setLogging(MU::Logger::LOUD) if $opts[:verbose]
 
-if !["groom", "chefrun", "vaults", "userdata", "awsmeta", "certs", "chefupgrade"].include?($opts[:mode])
-  Optimist::die(:mode, "--mode must be one of: groom, chefrun, awsmeta, vaults, certs, chefupgrade")
+$opts[:mode] = "groomeronly" if $opts[:mode] == "chefrun"
+
+if !["groom", "groomeronly", "vaults", "userdata", "awsmeta", "certs", "chefupgrade"].include?($opts[:mode])
+  Optimist::die(:mode, "--mode must be one of: groom, groomeronly, awsmeta, vaults, certs, chefupgrade")
 end
 if $opts[:platform] and !["linux", "windows"].include?($opts[:platform])
   Optimist::die(:platform, "--platform must be one of: linux, windows")
@@ -176,7 +178,7 @@ end
 exit 1 if !ok
 
 
-def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false)
+def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false, groomeronly: false)
   badnodes = []
   count = 0
   deploys.each { |muid|
@@ -196,6 +198,8 @@ def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false)
                 server.config["vault_access"].each { |v|
                   MU::Groomer::Chef.grantSecretAccess(mu_name, v['vault'], v['item'])
                 }
+              elsif groomeronly
+                server.groomer.run
               else
                 mommacat.groomNode(server.cloud_id, nodeclass, type, mu_name: mu_name)
               end
@@ -227,7 +231,7 @@ def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false)
   end
 end
 
-def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_output: $opts[:verbose], noop: false, chefrun: false, chef_runlist: nil)
+def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_output: $opts[:verbose], noop: false)
   badnodes = []
   count = 0
   deploys.each { |muid|
@@ -247,12 +251,6 @@ def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_
             next
           end
 
-          # Generate the command if attemting a chef run
-          if chefrun
-            cmd = serverobj.windows? ? "powershell -Command chef-client" : "chef-client || sudo chef-client"
-            cmd += " -o '#{chef_runlist}'" if chef_runlist
-          end
-
           MU.log "Running '#{cmd}' on #{nodename} (##{count})" if !print_output
 
           # Set Variables to catch the output and exit code of the execution
@@ -363,7 +361,7 @@ def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_
   }
 
   if badnodes.size > 0
-    cmd = "Chef" if $opts[:mode] == "chefrun"
+    cmd = "Chef" if $opts[:mode] == "groomeronly"
     if !print_output
       MU.log "Not all `#{cmd}` runs exited cleanly", MU::WARN, details: badnodes
     else
@@ -687,12 +685,13 @@ elsif $opts[:mode] == "vaults"
   reGroom(do_deploys, do_nodes, vaults_only: true)
 elsif $opts[:mode] == "chefupgrade"
   chefUpgrade(do_deploys, do_nodes)
-elsif $opts[:mode] == "chefrun"
+elsif $opts[:mode] == "groomeronly"
   print_output = $opts[:verbose] || do_nodes.size == 1
   if $opts[:override_chef_runlist]
-    runCommand(do_deploys, do_nodes, chef_runlist: $opts[:override_chef_runlist], chefrun: true, print_output: print_output)
+#    runCommand(do_deploys, do_nodes, chef_runlist: $opts[:override_chef_runlist], groomeronly: true, print_output: print_output)
   else
-    runCommand(do_deploys, do_nodes, chefrun: true, print_output: print_output)
+#    runCommand(do_deploys, do_nodes, groomeronly: true, print_output: print_output)
+    reGroom(do_deploys, do_nodes, groomeronly: true)
   end
 elsif $opts[:mode] == "userdata" or $opts[:mode] == "awsmeta"
 # Need Google equiv and to select nodes correctly based on what cloud they're in

cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.1.5'
-  s.date        = '2020-03-03'
+  s.version     = '3.1.6'
+  s.date        = '2020-03-20'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"

cookbooks/mu-activedirectory/resources/domain.rb

@@ -19,7 +19,7 @@
 attribute :restore_mode_password, :kind_of => String, :required => true
 attribute :site_name, :kind_of => String, :default => node['ad']['site_name'], :required => false
 attribute :computer_name, :kind_of => String, :default => node['ad']['computer_name']
-attribute :ntds_static_port, :kind_of => Fixnum, :default => node['ad']['ntds_static_port']
-attribute :ntfrs_static_port, :kind_of => Fixnum, :default => node['ad']['ntfrs_static_port']
-attribute :dfsr_static_port, :kind_of => Fixnum, :default => node['ad']['dfsr_static_port']
-attribute :netlogon_static_port, :kind_of => Fixnum, :default => node['ad']['netlogon_static_port']
+attribute :ntds_static_port, :kind_of => Integer, :default => node['ad']['ntds_static_port']
+attribute :ntfrs_static_port, :kind_of => Integer, :default => node['ad']['ntfrs_static_port']
+attribute :dfsr_static_port, :kind_of => Integer, :default => node['ad']['dfsr_static_port']
+attribute :netlogon_static_port, :kind_of => Integer, :default => node['ad']['netlogon_static_port']

cookbooks/mu-activedirectory/resources/domain_controller.rb

@@ -19,7 +19,7 @@
 attribute :restore_mode_password, :kind_of => String, :required => true
 attribute :site_name, :kind_of => String, :default => node['ad']['site_name'], :required => false
 attribute :computer_name, :kind_of => String, :default => node['ad']['computer_name']
-attribute :ntds_static_port, :kind_of => Fixnum, :default => node['ad']['ntds_static_port']
-attribute :ntfrs_static_port, :kind_of => Fixnum, :default => node['ad']['ntfrs_static_port']
-attribute :dfsr_static_port, :kind_of => Fixnum, :default => node['ad']['dfsr_static_port']
-attribute :netlogon_static_port, :kind_of => Fixnum, :default => node['ad']['netlogon_static_port']
+attribute :ntds_static_port, :kind_of => Integer, :default => node['ad']['ntds_static_port']
+attribute :ntfrs_static_port, :kind_of => Integer, :default => node['ad']['ntfrs_static_port']
+attribute :dfsr_static_port, :kind_of => Integer, :default => node['ad']['dfsr_static_port']
+attribute :netlogon_static_port, :kind_of => Integer, :default => node['ad']['netlogon_static_port']

cookbooks/mu-tools/recipes/eks.rb

@@ -160,8 +160,8 @@
 
   opento.uniq.each { |src|
     [:tcp, :udp, :icmp].each { |proto|
-      execute "iptables -I INPUT -p #{proto} -s #{src}" do
-        not_if "iptables -L -n | tr -s ' ' | grep -- '#{proto} -- #{src.sub(/\/32$/, "")}' > /dev/null"
+      execute "iptables -w 30 -I INPUT -p #{proto} -s #{src}" do
+        not_if "iptables -w 30 -L -n | tr -s ' ' | grep -- '#{proto} -- #{src.sub(/\/32$/, "")}' > /dev/null"
       end
     }
   }