AMBARI-25942: Useshell=False wherever subprocess module with shell=True is used

ambari-agent/conf/unix/agent-multiplier.py

@@ -20,6 +20,7 @@
 from ambari_commons import subprocess32
 import shutil
 from optparse import OptionParser
+import shlex
 
 
 class Host:
@@ -226,7 +227,7 @@ def create_host_name_script(self, host_name, host_name_script):
                "echo HOSTNAME"
     with open(str(host_name_script), "w+") as f:
       f.writelines(template.replace("HOSTNAME", host_name))
-    subprocess32.call("chmod +x %s" % host_name_script, shell=True)
+    subprocess32.call(shlex.split("chmod +x %s" % host_name_script), shell=False)
 
   def change_config(self, config_file, config_dict):
     """
@@ -321,21 +322,21 @@ def cmd_start(self):
     for host in self.hosts:
       cmd = "ambari-agent start --home %s" % (host.home_dir)
       os.environ['AMBARI_AGENT_CONF_DIR'] = os.path.join(host.home_dir, "etc/ambari-agent/conf")
-      subprocess32.call(cmd, shell=True, env=os.environ)
+      subprocess32.call(shlex.split(cmd), shell=False, env=os.environ)
 
   def cmd_stop(self):
     print "Stopping %d host(s)" % len(self.hosts)
     for host in self.hosts:
       cmd = "ambari-agent stop --home %s" % (host.home_dir)
       os.environ['AMBARI_AGENT_CONF_DIR'] = os.path.join(host.home_dir, "etc/ambari-agent/conf")
-      subprocess32.call(cmd, shell=True, env=os.environ)
+      subprocess32.call(shlex.split(cmd), shell=False, env=os.environ)
 
   def cmd_restart(self):
     print "Restarting %d host(s)" % len(self.hosts)
     for host in self.hosts:
       cmd = "ambari-agent restart --home %s" % (host.home_dir)
       os.environ['AMBARI_AGENT_CONF_DIR'] = os.path.join(host.home_dir, "etc/ambari-agent/conf")
-      subprocess32.call(cmd, shell=True, env=os.environ)
+      subprocess32.call(shlex.split(cmd), shell=False, env=os.environ)
 
   def cmd_status(self):
     print "Summary of Agent Status:"

ambari-agent/src/main/python/ambari_agent/Controller.py

@@ -37,6 +37,7 @@
 import security
 import ssl
 import AmbariConfig
+import shlex
 
 from ambari_agent.Heartbeat import Heartbeat
 from ambari_agent.Register import Register
@@ -633,12 +634,12 @@ def move_data_dir_mount_file(self):
         if os.path.exists(source_file) and not os.path.exists(destination_file):
           command = "mkdir -p %s" % os.path.dirname(destination_file)
           logger.info("Moving Data Dir Mount History file. Executing command: %s" % command)
-          return_code = subprocess32.call(command, shell=True)
+          return_code = subprocess32.call(shlex.split(command), shell=False)
           logger.info("Return code: %d" % return_code)
 
           command = "mv %s %s" % (source_file, destination_file)
           logger.info("Moving Data Dir Mount History file. Executing command: %s" % command)
-          return_code = subprocess32.call(command, shell=True)
+          return_code = subprocess32.call(shlex.split(command), shell=False)
           logger.info("Return code: %d" % return_code)
     except Exception, e:
       logger.error("Exception in move_data_dir_mount_file(). Error: {0}".format(str(e)))

ambari-agent/src/main/python/ambari_agent/PingPortListener.py

@@ -23,6 +23,7 @@
 import threading
 import socket
 from ambari_commons import subprocess32
+import shlex
 
 logger = logging.getLogger(__name__)
 FUSER_CMD = "fuser {0}/tcp 2>/dev/null | awk '{1}'"
@@ -51,10 +52,10 @@ def __init__(self, config):
 
 
   def run_os_command_in_shell(self, command):
-    process = subprocess32.Popen(command, stdout=subprocess32.PIPE,
+    process = subprocess32.Popen(shlex.split(command), stdout=subprocess32.PIPE,
               stdin=subprocess32.PIPE,
               stderr=subprocess32.PIPE,
-              shell=True)
+              shell=False)
     return process.communicate()
 
   def __del__(self):

ambari-agent/src/main/python/ambari_agent/hostname.py

@@ -24,6 +24,7 @@
 import logging
 import traceback
 import sys
+import shlex
 
 logger = logging.getLogger(__name__)
 
@@ -80,7 +81,10 @@ def public_hostname(config):
   try:
     if config.has_option('agent', 'public_hostname_script'):
       scriptname = config.get('agent', 'public_hostname_script')
-      output = subprocess32.Popen(scriptname, stdout=subprocess32.PIPE, stderr=subprocess32.PIPE, shell=True)
+      output = subprocess32.Popen(shlex.split(scriptname),
+                                  stdout=subprocess32.PIPE,
+                                  stderr=subprocess32.PIPE,
+                                  shell=False)
       out, err = output.communicate()
       if (0 == output.returncode and 0 != len(out.strip())):
         cached_public_hostname = out.strip().lower()

ambari-agent/src/main/python/ambari_agent/security.py

@@ -34,6 +34,7 @@
 import threading
 from ambari_stomp.adapter.websocket import WsConnection
 from socket import error as socket_error
+import shlex
 
 logger = logging.getLogger(__name__)
 
@@ -303,7 +304,7 @@ def genAgentCrtReq(self, keyname):
       p = subprocess32.Popen(generate_script, stdout=subprocess32.PIPE)
       p.communicate()
     else:
-      p = subprocess32.Popen([generate_script], shell=True,
+      p = subprocess32.Popen(shlex.split(generate_script), shell=False,
                            stdout=subprocess32.PIPE)
       p.communicate()
     # this is required to be 600 for security concerns.

ambari-common/src/main/python/resource_management/core/providers/mount.py

@@ -29,13 +29,14 @@
 from resource_management.core.base import Fail
 from resource_management.core.providers import Provider
 from resource_management.core.logger import Logger
+import shlex
 
 
 def get_mounted():
   """
   :return: Return a list of mount objects (dictionary type) that contain the device, mount point, and other options.
   """
-  p = Popen("mount", stdout=PIPE, stderr=STDOUT, shell=True)
+  p = Popen(shlex.split("mount"), stdout=PIPE, stderr=STDOUT, shell=False)
   out = p.communicate()[0]
   if p.wait() != 0:
     raise Fail("Getting list of mounts (calling mount) failed")

ambari-infra/ambari-infra-solr-client/src/main/python/migrationConfigGenerator.py

@@ -32,6 +32,7 @@
 import ConfigParser
 from subprocess import Popen, PIPE
 from random import randrange
+import shlex
 
 SOLR_SERVICE_NAME = 'AMBARI_INFRA_SOLR'
 SOLR_COMPONENT_NAME ='INFRA_SOLR'
@@ -138,7 +139,7 @@ def get_state_json_map(solr_urls, user='infra-solr', kerberos_enabled='false', k
   state_json_data={}
   request = GET_STATE_JSON_URL.format(get_random_solr_url(solr_urls))
   get_state_json_cmd=create_solr_api_request_command(request, user, kerberos_enabled, keytab, principal)
-  process = Popen(get_state_json_cmd, stdout=PIPE, stderr=PIPE, shell=True)
+  process = Popen(shlex.split(get_state_json_cmd), stdout=PIPE, stderr=PIPE, shell=False)
   out, err = process.communicate()
   if process.returncode != 0:
     logger.error(str(err))

ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py

@@ -37,6 +37,7 @@
 from subprocess import Popen, PIPE
 
 import solrDataManager as solr_data_manager
+import shlex
 
 HTTP_PROTOCOL = 'http'
 HTTPS_PROTOCOL = 'https'
@@ -843,7 +844,7 @@ def monitor_solr_async_request(options, config, status_request, request_id):
   async_request_timed_out = False
   while not async_request_finished:
     tries = tries + 1
-    process = Popen(request_status_json_cmd, stdout=PIPE, stderr=PIPE, shell=True)
+    process = Popen(shlex.split(request_status_json_cmd), stdout=PIPE, stderr=PIPE, shell=False)
     out, err = process.communicate()
     if process.returncode != 0:
       raise Exception("{0} command failed: {1}".format(request_status_json_cmd, str(err)))
@@ -893,7 +894,7 @@ def delete_collection(options, config, collection, solr_urls, response_data_map)
   request = DELETE_SOLR_COLLECTION_URL.format(solr_url, collection, async_id)
   logger.debug("Solr request: {0}".format(request))
   delete_collection_json_cmd=create_solr_api_request_command(request, config)
-  process = Popen(delete_collection_json_cmd, stdout=PIPE, stderr=PIPE, shell=True)
+  process = Popen(shlex.split(delete_collection_json_cmd), stdout=PIPE, stderr=PIPE, shell=False)
   out, err = process.communicate()
   if process.returncode != 0:
     raise Exception("{0} command failed: {1}".format(delete_collection_json_cmd, str(err)))
@@ -910,7 +911,7 @@ def create_collection(options, config, solr_urls, collection, config_set, shards
   request = CREATE_SOLR_COLLECTION_URL.format(get_random_solr_url(solr_urls, options), collection, config_set, shards, replica, max_shards_per_node)
   logger.debug("Solr request: {0}".format(request))
   create_collection_json_cmd=create_solr_api_request_command(request, config)
-  process = Popen(create_collection_json_cmd, stdout=PIPE, stderr=PIPE, shell=True)
+  process = Popen(shlex.split(create_collection_json_cmd), stdout=PIPE, stderr=PIPE, shell=False)
   out, err = process.communicate()
   if process.returncode != 0:
     raise Exception("{0} command failed: {1}".format(create_collection_json_cmd, str(err)))
@@ -925,7 +926,7 @@ def reload_collection(options, config, solr_urls, collection):
   request = RELOAD_SOLR_COLLECTION_URL.format(get_random_solr_url(solr_urls, options), collection)
   logger.debug("Solr request: {0}".format(request))
   reload_collection_json_cmd=create_solr_api_request_command(request, config)
-  process = Popen(reload_collection_json_cmd, stdout=PIPE, stderr=PIPE, shell=True)
+  process = Popen(shlex.split(reload_collection_json_cmd), stdout=PIPE, stderr=PIPE, shell=False)
   out, err = process.communicate()
   if process.returncode != 0:
     raise Exception("{0} command failed: {1}".format(reload_collection_json_cmd, str(err)))
@@ -962,7 +963,7 @@ def get_replica_index_size(config, core_url, replica):
   request = CORE_DETAILS_URL.format(core_url)
   logger.debug("Solr request: {0}".format(request))
   get_core_detaul_json_cmd=create_solr_api_request_command(request, config)
-  process = Popen(get_core_detaul_json_cmd, stdout=PIPE, stderr=PIPE, shell=True)
+  process = Popen(shlex.split(get_core_detaul_json_cmd), stdout=PIPE, stderr=PIPE, shell=False)
   out, err = process.communicate()
   if process.returncode != 0:
     raise Exception("{0} command failed: {1}".format(get_core_detaul_json_cmd, str(err)))
@@ -980,7 +981,7 @@ def delete_znode(options, config, znode):
   logger.debug("Solr cli command: {0}".format(solr_cli_command))
   sys.stdout.write('Deleting znode {0} ... '.format(znode))
   sys.stdout.flush()
-  process = Popen(solr_cli_command, stdout=PIPE, stderr=PIPE, shell=True)
+  process = Popen(shlex.split(solr_cli_command), stdout=PIPE, stderr=PIPE, shell=False)
   out, err = process.communicate()
   if process.returncode != 0:
     sys.stdout.write(colors.FAIL + 'FAILED\n' + colors.ENDC)
@@ -999,7 +1000,7 @@ def copy_znode(options, config, copy_src, copy_dest, copy_from_local=False, copy
   logger.debug("Solr cli command: {0}".format(solr_cli_command))
   sys.stdout.write('Transferring data from {0} to {1} ... '.format(copy_src, copy_dest))
   sys.stdout.flush()
-  process = Popen(solr_cli_command, stdout=PIPE, stderr=PIPE, shell=True)
+  process = Popen(shlex.split(solr_cli_command), stdout=PIPE, stderr=PIPE, shell=False)
   out, err = process.communicate()
   if process.returncode != 0:
     sys.stdout.write(colors.FAIL + 'FAILED\n' + colors.ENDC)
@@ -1031,7 +1032,7 @@ def list_collections(options, config, output_file, include_number_of_docs=False)
     logger.debug("Solr cli command: {0}".format(solr_cli_command))
     sys.stdout.write('Dumping collections data to {0} ... '.format(output_file))
     sys.stdout.flush()
-    process = Popen(solr_cli_command, stdout=PIPE, stderr=PIPE, shell=True)
+    process = Popen(shlex.split(solr_cli_command), stdout=PIPE, stderr=PIPE, shell=False)
     out, err = process.communicate()
     if process.returncode != 0:
       sys.stdout.write(colors.FAIL + 'FAILED\n' + colors.ENDC)

ambari-metrics/ambari-metrics-host-monitoring/src/main/python/psutil/psutil/_pssunos.py

@@ -18,6 +18,7 @@
 from psutil._compat import namedtuple, PY3
 import _psutil_posix
 import _psutil_sunos as cext
+import shlex
 
 
 __extra__all__ = ["CONN_IDLE", "CONN_BOUND"]
@@ -433,8 +434,10 @@ def _get_unix_sockets(self, pid):
         # TODO: rewrite this in C (...but the damn netstat source code
         # does not include this part! Argh!!)
         cmd = "pfiles %s" % pid
-        p = subprocess32.Popen(cmd, shell=True, stdout=subprocess32.PIPE,
-                             stderr=subprocess32.PIPE)
+        p = subprocess32.Popen(shlex.split(cmd),
+                               shell=False,
+                               stdout=subprocess32.PIPE,
+                               stderr=subprocess32.PIPE)
         stdout, stderr = p.communicate()
         if PY3:
             stdout, stderr = [x.decode(sys.stdout.encoding)

ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/phoenix_utils.py

@@ -22,6 +22,7 @@
 import os
 import fnmatch
 import subprocess
+import shlex
 
 def find(pattern, classPaths):
     paths = classPaths.split(os.pathsep)
@@ -62,7 +63,7 @@ def which(file):
 def findClasspath(file):
     aPath = which(file)
     command = "%s%s" %(aPath, ' classpath')
-    return subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).stdout.read()
+    return subprocess.Popen(shlex.split(command), shell=False, stdout=subprocess.PIPE).stdout.read()
 
 def setPath():
     PHOENIX_CLIENT_JAR_PATTERN = "phoenix-*-client.jar"

ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/sqlline.py

@@ -25,6 +25,7 @@
 import phoenix_utils
 import atexit
 import fnmatch
+import shlex
 
 # Set JAVA_HOME or JDK_HOME before running sqlline.py
 # Example: ./sqlline.py localhost:61181:/ams-hbase-unsecure
@@ -95,7 +96,7 @@ def find_java():
 
 print 'java command: %s' % str(java_cmd)
 
-childProc = subprocess.Popen(java_cmd, shell=True)
+childProc = subprocess.Popen(shlex.split(java_cmd), shell=False)
 #Wait for child process exit
 (output, error) = childProc.communicate()
 returncode = childProc.returncode

ambari-server/src/main/python/ambari_server/serverSetup.py

@@ -26,6 +26,7 @@
 from ambari_commons import subprocess32
 import getpass
 import logging
+import shlex
 
 from ambari_commons.exceptions import FatalException
 from ambari_commons.firewall import Firewall
@@ -846,11 +847,11 @@ def adjust_jce_permissions(self, jdk_path):
 
     cmd = " && ".join(cmds)
 
-    process = subprocess32.Popen(cmd,
+    process = subprocess32.Popen(shlex.split(cmd),
                            stdout=subprocess32.PIPE,
                            stdin=subprocess32.PIPE,
                            stderr=subprocess32.PIPE,
-                           shell=True
+                           shell=False
                            )
     (stdoutdata, stderrdata) = process.communicate()
 
@@ -1281,11 +1282,11 @@ def check_ambari_java_version_is_valid(java_home, java_bin, min_version, propert
   print 'Check JDK version for Ambari Server...'
   try:
     command = JDK_VERSION_CHECK_CMD.format(os.path.join(java_home, 'bin', java_bin))
-    process = subprocess32.Popen(command,
+    process = subprocess32.Popen(shlex.split(command),
                                stdout=subprocess32.PIPE,
                                stdin=subprocess32.PIPE,
                                stderr=subprocess32.PIPE,
-                               shell=True
+                               shell=False
                                )
     (out, err) = process.communicate()
     if process.returncode != 0:

ambari-server/src/main/python/ambari_server/utils.py

@@ -28,6 +28,7 @@
 import logging
 import platform
 import xml.etree.ElementTree as ET
+import shlex
 
 from ambari_commons import OSConst,OSCheck
 from ambari_commons.logging_utils import print_error_msg
@@ -272,11 +273,11 @@ def get_postgre_hba_dir(OS_FAMILY):
     pg_hba_init_basename = os.path.basename(get_pg_hba_init_files())
     # Get postgres_data location (default: /var/lib/pgsql/data)
     cmd = "alias basename='echo {0}; true' ; alias exit=return; source {1} status &>/dev/null; echo $PGDATA".format(pg_hba_init_basename, get_pg_hba_init_files())
-    p = subprocess32.Popen(cmd,
+    p = subprocess32.Popen(shlex.split(cmd),
                          stdout=subprocess32.PIPE,
                          stdin=subprocess32.PIPE,
                          stderr=subprocess32.PIPE,
-                         shell=True)
+                         shell=False)
     (PG_HBA_ROOT, err) = p.communicate()
 
     if PG_HBA_ROOT and len(PG_HBA_ROOT.strip()) > 0:

ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py

@@ -20,6 +20,7 @@
 from resource_management.core.resources.system import Execute, Directory
 from resource_management.core.exceptions import Fail
 from resource_management.core.logger import Logger
+import shlex
 
 import hawq_constants
 
@@ -83,7 +84,11 @@ def exec_ssh_cmd(hostname, cmd):
   # Only gpadmin should be allowed to run command via ssh, thus not exposing user as a parameter
   cmd = "su - {0} -c \"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {1} \\\"{2} \\\" \"".format(hawq_constants.hawq_user, hostname, cmd)
   Logger.info("Command executed: {0}".format(cmd))
-  process = subprocess32.Popen(cmd, stdout=subprocess32.PIPE, stdin=subprocess32.PIPE, stderr=subprocess32.PIPE, shell=True)
+  process = subprocess32.Popen(shlex.split(cmd),
+                               stdout=subprocess32.PIPE,
+                               stdin=subprocess32.PIPE,
+                               stderr=subprocess32.PIPE,
+                               shell=False)
   (stdout, stderr) = process.communicate()
   return process.returncode, stdout, stderr
 

ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf_utils.py

@@ -21,6 +21,7 @@
 import urllib2
 import urllib
 from ambari_commons import subprocess32
+import shlex
 
 def makeHTTPCall(url, header={}, body=None):
   # timeout in seconds
@@ -45,5 +46,5 @@ def makeHTTPCall(url, header={}, body=None):
 
 
 def runLocalCmd(cmd):
-  return subprocess32.call(cmd, shell=True)  
+  return subprocess32.call(shlex.split(cmd), shell=False)