Changes for supporting ppc64le

.gitignore

@@ -42,9 +42,10 @@ bin/*
 
 # config/
 config/apache
+config/cable.yml
 config/cockpit
 config/database.yml*
-config/cable.yml
+config/messaging.yml
 config/vmdb.yml.db
 
 config/initializers/*.local.rb

.travis.yml

@@ -1,4 +1,7 @@
-dist: xenial
+dist: bionic
+arch:
+- amd64
+- ppc64le
 language: ruby
 rvm:
 - 2.5.7
@@ -17,8 +20,8 @@ env:
   - TEST_SUITE=brakeman
 matrix:
   fast_finish: true
-addons:
-  postgresql: '10'
+services:
+- postgresql
 before_install:
 - source ${TRAVIS_BUILD_DIR}/tools/ci/before_install.sh
 before_script:

CHANGELOG.md

@@ -4,6 +4,31 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 
+# Unreleased as of Sprint 134 ending 2020-04-13
+
+### Fixed
+
+* Fix messaging_client exception if no ENV and no yaml are present [(#20062)](https://github.com/ManageIQ/manageiq/pull/20062)
+* Fix a bug when running Rbac on VimPerformanceDaily [(#20058)](https://github.com/ManageIQ/manageiq/pull/20058)
+* Fix improperly addressed rubocop warning [(#20044)](https://github.com/ManageIQ/manageiq/pull/20044)
+* Only show catalog types for supported EMSs [(#20039)](https://github.com/ManageIQ/manageiq/pull/20039)
+* Fix storage profile ems foreign key [(#20038)](https://github.com/ManageIQ/manageiq/pull/20038)
+
+# Unreleased as of Sprint 133 ending 2020-03-30
+
+### Added
+
+* Add additional privilege escalation methods [(#20019)](https://github.com/ManageIQ/manageiq/pull/20019)
+* Add kafka connection info to worker containers [(#20000)](https://github.com/ManageIQ/manageiq/pull/20000)
+* Add kafka as an option for prototype.queue_type [(#19984)](https://github.com/ManageIQ/manageiq/pull/19984)
+* Consolidate existing server setting CLI tools [(#19848)](https://github.com/ManageIQ/manageiq/pull/19848)
+
+### Fixed
+
+* Remove the journald logger to resolve segfaults [(#20004)](https://github.com/ManageIQ/manageiq/pull/20004)
+* Don't queue things that need to run on the same worker container [(#19956)](https://github.com/ManageIQ/manageiq/pull/19956)
+* Embedded ansible provider should allow creation in maintenance zone [(#19947)](https://github.com/ManageIQ/manageiq/pull/19947)
+
 # Unreleased as of Sprint 132 ending 2020-03-16
 
 ### Added

Gemfile

@@ -31,7 +31,7 @@ gem "acts_as_tree",                   "~>2.7" # acts_as_tree needs to be require
 gem "ancestry",                       "~>3.0.7",       :require => false
 gem "aws-sdk-s3",                     "~>1.0",         :require => false # For FileDepotS3
 gem "bcrypt",                         "~> 3.1.10",     :require => false
-gem "bundler",                        ">=1.15",        :require => false
+gem "bundler",                        "~> 2.1.4",      :require => false
 gem "byebug",                                          :require => false
 gem "color",                          "~>1.8"
 gem "config",                         "~>2.2", ">=2.2.1", :require => false
@@ -53,6 +53,7 @@ gem "manageiq-loggers",               "~>0.5.0",       :require => false
 gem "manageiq-messaging",             "~>0.1.4",       :require => false
 gem "manageiq-password",              "~>0.3",         :require => false
 gem "manageiq-postgres_ha_admin",     "~>3.1",         :require => false
+gem "manageiq-ssh-util",              "~>0.1.0",       :require => false
 gem "memoist",                        "~>0.15.0",      :require => false
 gem "mime-types",                     "~>3.0",         :path => File.expand_path("mime-types-redirector", __dir__)
 gem "money",                          "~>6.13.5",      :require => false
@@ -138,7 +139,7 @@ group :redfish, :manageiq_default do
 end
 
 group :qpid_proton, :optional => true do
-  gem "qpid_proton",                    "~>0.26.0",      :require => false
+  gem "qpid_proton",                    "~>0.30.0",      :require => false
 end
 
 group :systemd, :optional => true do

app/models/authenticator/base.rb

@@ -137,7 +137,18 @@ def authorize(taskid, username, *args)
           end
 
           user.lastlogon = Time.now.utc
-          user.save!
+          if user.new_record?
+            User.with_lock do
+              user.save!
+            rescue ActiveRecord::RecordInvalid # Try update when catching create race condition.
+              userid, user = find_or_initialize_user(identity, username)
+              update_user_attributes(user, userid, identity)
+              user.miq_groups = matching_groups
+              user.save!
+            end
+          else
+            user.save!
+          end
 
           _log.info("Authorized User: [#{user.userid}]")
           task.userid = user.userid

app/models/conversion_host.rb

@@ -78,9 +78,9 @@ def verify_credentials(auth_type = 'v2v', options = {})
       Net::SSH.start(host, auth.userid, ssh_options) { |ssh| ssh.exec!('uname -a') }
     end
   rescue Net::SSH::AuthenticationFailed => err
-    raise MiqException::MiqInvalidCredentialsError, _("Incorrect credentials - %{error_message}") % {:error_message => err.message}
+    raise err, _("Incorrect credentials - %{error_message}") % {:error_message => err.message}
   rescue Net::SSH::HostKeyMismatch => err
-    raise MiqException::MiqSshUtilHostKeyMismatch, _("Host key mismatch - %{error_message}") % {:error_message => err.message}
+    raise err, _("Host key mismatch - %{error_message}") % {:error_message => err.message}
   rescue Exception => err
     raise _("Unknown error - %{error_message}") % {:error_message => err.message}
   else
@@ -155,8 +155,8 @@ def ipaddress(family = 'ipv4')
   #
   # @return [Integer] length of data written to file
   #
-  # @raise [MiqException::MiqInvalidCredentialsError] if conversion host credentials are invalid
-  # @raise [MiqException::MiqSshUtilHostKeyMismatch] if conversion host key has changed
+  # @raise [Net::SSH::AuthenticationFailed] if conversion host credentials are invalid
+  # @raise [Net::SSH::HostKeyMismatch] if conversion host key has changed
   # @raise [JSON::GeneratorError] if limits hash can't be converted to JSON
   # @raise [StandardError] if any other problem happens
   def apply_task_limits(task_id, limits = {})
@@ -165,7 +165,7 @@ def apply_task_limits(task_id, limits = {})
       command = AwesomeSpawn.build_command_line("mv", ["/tmp/#{task_id}-limits.json", "/var/lib/uci/#{task_id}/limits.json"])
       ssu.shell_exec(command, nil, nil, nil)
     end
-  rescue MiqException::MiqInvalidCredentialsError, MiqException::MiqSshUtilHostKeyMismatch => err
+  rescue Net::SSH::AuthenticationFailed, Net::SSH::HostKeyMismatch => err
     raise "Failed to connect and apply limits for task '#{task_id}' with [#{err.class}: #{err}]"
   rescue JSON::GeneratorError => err
     raise "Could not generate JSON from limits '#{limits}' with [#{err.class}: #{err}]"
@@ -180,8 +180,8 @@ def apply_task_limits(task_id, limits = {})
   #
   # @return [Integer] length of data written to conversion options file
   #
-  # @raise [MiqException::MiqInvalidCredentialsError] if conversion host credentials are invalid
-  # @raise [MiqException::MiqSshUtilHostKeyMismatch] if conversion host key has changed
+  # @raise [Net::SSH::AuthenticationFailed] if conversion host credentials are invalid
+  # @raise [Net::SSH::HostKeyMismatch] if conversion host key has changed
   # @raise [JSON::GeneratorError] if limits hash can't be converted to JSON
   # @raise [StandardError] if any other problem happens
   def prepare_conversion(task_id, conversion_options)
@@ -197,7 +197,7 @@ def prepare_conversion(task_id, conversion_options)
       command = AwesomeSpawn.build_command_line("mv", ["/tmp/#{task_id}-input.json", "/var/lib/uci/#{task_id}/input.json"])
       ssu.shell_exec(command, nil, nil, nil)
     end
-  rescue MiqException::MiqInvalidCredentialsError, MiqException::MiqSshUtilHostKeyMismatch => err
+  rescue Net::SSH::AuthenticationFailed, Net::SSH::HostKeyMismatch => err
     raise "Failed to connect and prepare conversion for task '#{task_id}' with [#{err.class}: #{err}]"
   rescue JSON::GeneratorError => err
     raise "Could not generate JSON for task '#{task_id}' from options '#{filtered_options}' with [#{err.class}: #{err}]"
@@ -210,14 +210,14 @@ def prepare_conversion(task_id, conversion_options)
   #
   # @return [Boolean] true if the file can be retrieved and parsed, false otherwise
   #
-  # @raise [MiqException::MiqInvalidCredentialsError] if conversion host credentials are invalid
-  # @raise [MiqException::MiqSshUtilHostKeyMismatch] if conversion host key has changed
+  # @raise [Net::SSH::AuthenticationFailed] if conversion host credentials are invalid
+  # @raise [Net::SSH::HostKeyMismatch] if conversion host key has changed
   # @raise [JSON::ParserError] if file cannot be parsed as JSON
   def luks_keys_vault_valid?
     luks_keys_vault_json = connect_ssh { |ssu| ssu.get_file("/root/.v2v_luks_keys_vault.json", nil) }
     JSON.parse(luks_keys_vault_json)
     true
-  rescue MiqException::MiqInvalidCredentialsError, MiqException::MiqSshUtilHostKeyMismatch => err
+  rescue Net::SSH::AuthenticationFailed, Net::SSH::HostKeyMismatch => err
     raise "Failed to connect and retrieve LUKS keys vault from file '/root/.v2v_luks_keys_vault.json' with [#{err.class}: #{err}]"
   rescue JSON::ParserError
     raise "Could not parse conversion state data from file '/root/.v2v_luks_keys_vault.json': #{json_state}"
@@ -266,7 +266,7 @@ def run_conversion(task_id, conversion_options)
     filtered_options = filter_options(conversion_options)
     prepare_conversion(task_id, conversion_options)
     connect_ssh { |ssu| ssu.shell_exec(build_podman_command(task_id, conversion_options), nil, nil, nil) }
-  rescue MiqException::MiqInvalidCredentialsError, MiqException::MiqSshUtilHostKeyMismatch => err
+  rescue Net::SSH::AuthenticationFailed, Net::SSH::HostKeyMismatch => err
     raise "Failed to connect and run conversion using options #{filtered_options} with [#{err.class}: #{err}]"
   rescue => err
     raise "Starting conversion for task '#{task_id}' failed on '#{resource.name}' with [#{err.class}: #{err}]"
@@ -313,7 +313,7 @@ def kill_virtv2v(task_id, signal = 'TERM')
   def get_conversion_state(task_id)
     json_state = connect_ssh { |ssu| ssu.get_file("/var/lib/uci/#{task_id}/state.json", nil) }
     JSON.parse(json_state)
-  rescue MiqException::MiqInvalidCredentialsError, MiqException::MiqSshUtilHostKeyMismatch => err
+  rescue Net::SSH::AuthenticationFailed, Net::SSH::HostKeyMismatch => err
     raise "Failed to connect and retrieve conversion state data from file '/var/lib/uci/#{task_id}/state.json' with [#{err.class}: #{err}]"
   rescue JSON::ParserError
     raise "Could not parse conversion state data from file '/var/lib/uci/#{task_id}/state.json': #{json_state}"
@@ -408,12 +408,12 @@ def filter_options(options)
     options.clone.tap { |h| h.each { |k, _v| h[k] = "__FILTERED__" if ignore.any? { |i| k.to_s.end_with?(i) } } }
   end
 
-  # Connect to the conversion host using the MiqSshUtil wrapper using the authentication
+  # Connect to the conversion host using the ManageIQ::SSH::Util wrapper using the authentication
   # parameters appropriate for that type of resource.
   #
   def connect_ssh
-    require 'MiqSshUtil'
-    MiqSshUtil.shell_with_su(*miq_ssh_util_args) do |ssu, _shell|
+    require 'manageiq-ssh-util'
+    ManageIQ::SSH::Util.shell_with_su(*miq_ssh_util_args) do |ssu, _shell|
       yield(ssu)
     end
   rescue Exception => e

app/models/conversion_host/configurations.rb

@@ -54,6 +54,7 @@ def enable_queue(params, auth_user = nil)
       params = params.symbolize_keys
       resource = params.delete(:resource)
 
+      raise "#{resource.class.name.demodulize} '#{resource.name}' doesn't have a hostname or IP address in inventory" if resource.hostname.nil? && resource.ipaddresses.empty?
       raise "the resource '#{resource.name}' is already configured as a conversion host" if ConversionHost.exists?(:resource => resource)
 
       params[:resource_id] = resource.id
@@ -113,12 +114,13 @@ def disable_queue(auth_user = nil)
 
   def disable(_params = nil, _auth_user = nil)
     resource_info = "type=#{resource.class.name} id=#{resource.id}"
-    _log.debug("Disabling a conversion_host #{resource_info}")
+    raise "There are active migration tasks running on this conversion host" if active_tasks.present?
 
+    _log.debug("Disabling a conversion_host #{resource_info}")
     disable_conversion_host_role
     destroy!
   rescue StandardError => error
-    raise
+    raise error
   ensure
     self.class.notify_configuration_result('disable', error.nil?, resource_info)
   end

app/models/custom_button.rb

@@ -3,7 +3,7 @@ class CustomButton < ApplicationRecord
 
   scope :with_array_order, lambda { |ids, column = :id, column_type = :bigint|
     order = sanitize_sql_array(["array_position(ARRAY[?]::#{column_type}[], #{table_name}.#{column}::#{column_type})", ids])
-    order(order)
+    order(Arel.sql(order))
   }
 
   serialize :options, Hash

app/models/dialog_field_importer.rb

@@ -34,7 +34,7 @@ def import_field(dialog_field_attributes, export_version = DialogImportService::
     elsif dialog_field_attributes["type"].nil?
       dialog_field_attributes.delete("dialog_field_responders")
       dialog_field_attributes.delete("resource_action")
-      DialogField.create(dialog_field_attributes)
+      DialogField.create!(dialog_field_attributes)
     else
       raise InvalidDialogFieldTypeError
     end

app/models/ems_event.rb

@@ -25,22 +25,22 @@ def self.bottleneck_event_groups
   end
 
   def self.add_queue(meth, ems_id, event)
-    if Settings.prototype.queue_type == 'artemis'
-      MiqQueue.artemis_client('event_handler').publish_topic(
-        :service => "events",
+    unless MiqQueue.messaging_type == "miq_queue"
+      MiqQueue.messaging_client('event_handler')&.publish_topic(
+        :service => "manageiq.ems-events",
         :sender  => ems_id,
         :event   => event[:event_type],
         :payload => event
       )
-    else
-      MiqQueue.submit_job(
-        :service     => "event",
-        :target_id   => ems_id,
-        :class_name  => "EmsEvent",
-        :method_name => meth,
-        :args        => [event],
-      )
     end
+
+    MiqQueue.submit_job(
+      :service     => "event",
+      :target_id   => ems_id,
+      :class_name  => "EmsEvent",
+      :method_name => meth,
+      :args        => [event]
+    )
   end
 
   def self.add(ems_id, event_hash)

app/models/ext_management_system.rb

@@ -17,36 +17,39 @@ def self.supported_types
     supported_subclasses.collect(&:ems_type)
   end
 
-  def self.leaf_subclasses
-    descendants.select { |d| d.subclasses.empty? }
+  def self.supported_subclasses
+    leaf_subclasses.select(&:permitted?)
   end
 
-  def self.supported_subclasses
-    subclasses.flat_map do |s|
-      s.subclasses.empty? ? s : s.supported_subclasses
-    end
+  def self.permitted?
+    Vmdb::PermissionStores.instance.supported_ems_type?(ems_type)
   end
+  delegate :permitted?, :to => :class
 
   def self.supported_types_and_descriptions_hash
-    supported_subclasses.each_with_object({}) do |klass, hash|
-      if Vmdb::PermissionStores.instance.supported_ems_type?(klass.ems_type)
-        hash[klass.ems_type] = klass.description
-      end
-    end
+    supported_subclasses.each_with_object({}) { |klass, hash| hash[klass.ems_type] = klass.description }
   end
 
   def self.api_allowed_attributes
     %w[]
   end
 
   def self.supported_types_for_create
-    leaf_subclasses.select(&:supported_for_create?)
+    supported_subclasses.select(&:supported_for_create?)
+  end
+
+  def self.supported_types_for_catalog
+    supported_subclasses.select(&:supported_for_catalog?)
   end
 
   def self.supported_for_create?
     !reflections.include?("parent_manager")
   end
 
+  def self.supported_for_catalog?
+    catalog_types.present?
+  end
+
   def self.provider_create_params
     supported_types_for_create.each_with_object({}) do |ems_type, create_params|
       create_params[ems_type.name] = ems_type.params_for_create if ems_type.respond_to?(:params_for_create)
@@ -540,6 +543,9 @@ def last_refresh_status
     end
   end
 
+  # Queue an EMS refresh using +opts+. Credentials must exist, and the
+  # authentication status must be ok, otherwise an error is raised.
+  #
   def refresh_ems(opts = {})
     if missing_credentials?
       raise _("no Provider credentials defined")
@@ -550,6 +556,18 @@ def refresh_ems(opts = {})
     EmsRefresh.queue_refresh(self, nil, opts)
   end
 
+  alias queue_refresh refresh_ems
+
+  # Execute an EMS refresh immediately. Credentials must exist, and the
+  # authentication status must be ok, otherwise an error is raised.
+  #
+  def refresh
+    raise _("no Provider credentials defined") if missing_credentials?
+    raise _("Provider failed last authentication check") unless authentication_status_ok?
+
+    EmsRefresh.refresh(self)
+  end
+
   def self.ems_infra_discovery_types
     @ems_infra_discovery_types ||= %w(virtualcenter scvmm rhevm openstack_infra)
   end

app/models/filesystem.rb

@@ -46,7 +46,7 @@ def self.process_sub_xml(xmlNode, path, options = {})
         nh = e.attributes.to_h
 
         nh[:base_name] = nh[:name]
-        nh[:name] = File.join(path, nh[:name])
+        nh[:name] = nh[:fqname]
         nh[:rsc_type] = e.name
         nh.delete(:fqname)
         nh[:mtime] = Time.parse(nh[:mtime])