CA-388295: Fix bugs from py2->py3 conversion in perfmon and hfx_filename

Signed-off-by: Stephen Cheng <[email protected]>

scripts/hfx_filename

@@ -28,18 +28,14 @@ def rpc(session_id, request):
         headers = [
           "POST %s?session_id=%s HTTP/1.0" % (db_url, session_id),
           "Connection:close",
-          "content-length:%d" % (len(request)),
+          "content-length:%d" % (len(request.encode('utf-8'))),
           ""
         ]
-        #print "Sending HTTP request:"
         for h in headers:
-          s.send("%s\r\n" % h)
-          #print "%s\r\n" % h,
-        s.send(request)
+          s.send((h + "\r\n").encode('utf-8'))
+        s.send(request.encode('utf-8'))
 
-        result = s.recv(1024)
-        #print "Received HTTP response:"
-        #print result
+        result = s.recv(1024).decode('utf-8')
         if "200 OK" not in result:
           print("Expected an HTTP 200, got %s" % result, file=sys.stderr)
           return
@@ -57,11 +53,11 @@ def rpc(session_id, request):
 def parse_string(txt):
     prefix = "<value><array><data><value>success</value><value>"
     if not txt.startswith(prefix):
-        raise "Unable to parse string response"
+        raise Exception("Unable to parse string response")
     txt = txt[len(prefix):]
     suffix = "</value></data></array></value>"
     if not txt.endswith(suffix):
-        raise "Unable to parse string response"
+        raise Exception("Unable to parse string response")
     txt = txt[:len(txt)-len(suffix)]
     return txt
 

scripts/import_file.py

@@ -0,0 +1,24 @@
+# Used for importing a non-".py" file as a module
+
+import sys
+import os
+
+def import_from_file(module_name, file_path):
+    """Import a file as a module"""
+    # Only for python3, but CI has python2 pytest check, so add this line
+    if sys.version_info.major == 2:
+        return None
+    from importlib import machinery, util
+    loader = machinery.SourceFileLoader(module_name, file_path)
+    spec = util.spec_from_loader(module_name, loader)
+    assert spec
+    assert spec.loader
+    module = util.module_from_spec(spec)
+    # Probably a good idea to add manually imported module stored in sys.modules
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    return module
+
+def get_module(module_name, file_path):
+    testdir = os.path.dirname(__file__)
+    return import_from_file(module_name, testdir + file_path)

scripts/perfmon

@@ -313,7 +313,7 @@ class RRDUpdates:
         print_debug("Calling http://localhost/rrd_updates?%s" % paramstr)
 
         sock = urllib.request.urlopen("http://localhost/rrd_updates?%s" % paramstr)
-        xmlsource = sock.read()
+        xmlsource = sock.read().decode('utf-8')
         sock.close()
 
         # Use sax rather than minidom and save Vvvast amounts of time and memory.
@@ -380,7 +380,7 @@ def get_percent_mem_usage(ignored):
         memlist = memfd.readlines()
         memfd.close()
         memdict = [ m.split(':', 1) for m in memlist ]
-        memdict = dict([(k.strip(), float(re.search('\d+', v.strip()).group(0))) for (k,v) in memdict])
+        memdict = dict([(k.strip(), float(re.search(r'\d+', v.strip()).group(0))) for (k,v) in memdict])
         # We consider the sum of res memory and swap in use as the hard demand
         # of mem usage, it is bad if this number is beyond the physical mem, as
         # in such case swapping is obligatory rather than voluntary, hence
@@ -1067,7 +1067,8 @@ def main():
             vm_uuid_list = rrd_updates.get_uuid_list_by_objtype('vm')
 
             # Remove any monitors for VMs no longer listed in rrd_updates page
-            for uuid in vm_mon_lookup:
+            # We use .pop() inside the loop, use list(dict_var.keys()):
+            for uuid in list(vm_mon_lookup.keys()):
                 if uuid not in vm_uuid_list:
                     vm_mon_lookup.pop(uuid)
 
@@ -1103,7 +1104,8 @@ def main():
             print_debug("sr_uuid_list = %s" % sr_uuid_list)
 
             # Remove monitors for SRs no longer listed in the rrd_updates page
-            for uuid in sr_mon_lookup:
+            # We use .pop() inside the loop, use list(dict_var.keys()):
+            for uuid in list(sr_mon_lookup.keys()):
                 if uuid not in sr_uuid_list:
                     sr_mon_lookup.pop(uuid)
             # Create monitors for SRs that have just appeared in rrd_updates page

scripts/test_hfx_filename.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# unittest for hfx_filename
+
+import unittest
+from mock import MagicMock, patch, call
+import sys
+from import_file import get_module
+
+
+# mock modules to avoid dependencies
+sys.modules["XenAPI"] = MagicMock()
+
+hfx_filename = get_module("hfx_filename", "/hfx_filename")
+
+@unittest.skipIf(sys.version_info < (3, 0), reason="requires python3")
+@patch("socket.socket")
+class TestRpc(unittest.TestCase):
+
+    def test_rpc(self, mock_socket):
+        mock_connected_socket = MagicMock()
+        mock_socket.return_value = mock_connected_socket
+
+        recv_data = b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHelloWorld"
+        mock_connected_socket.recv.return_value = recv_data
+
+        session_id = 0
+        request = "socket request"
+        body = hfx_filename.rpc(session_id, request)
+
+        # Assert that the socket methods were called as expected
+        expected_data = [
+            b"POST /remote_db_access?session_id=0 HTTP/1.0\r\n",
+            b"Connection:close\r\n",
+            b"content-length:14\r\n",
+            b"\r\n",
+            b"socket request"
+        ]
+        mock_connected_socket.send.assert_has_calls([call(data) for data in expected_data])
+
+        expected_return = "HelloWorld"
+        self.assertEqual(expected_return, body)
+
+    def test_rpc_international_character(self, mock_socket):
+        mock_connected_socket = MagicMock()
+        mock_socket.return_value = mock_connected_socket
+
+        recv_data = b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHelloWorld"
+        mock_connected_socket.recv.return_value = recv_data
+
+        session_id = 0
+        # Use international character"socket 请求" as request
+        # Not using literal string is for passing python2 check
+        request = "socket 请求"
+        body = hfx_filename.rpc(session_id, request)
+
+        # Assert that the socket methods were called as expected
+        expected_data = [
+            b"POST /remote_db_access?session_id=0 HTTP/1.0\r\n",
+            b"Connection:close\r\n",
+            b"content-length:13\r\n",
+            b"\r\n",
+            request.encode('utf-8')
+        ]
+        mock_connected_socket.send.assert_has_calls([call(data) for data in expected_data])
+
+        expected_return = "HelloWorld"
+        self.assertEqual(expected_return, body)
+
+    def test_db_get_uuid(self, mock_socket):
+        mock_connected_socket = MagicMock()
+        mock_socket.return_value = mock_connected_socket
+
+        header = "HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n"
+        body = "<value><array><data><value>success</value><value>HelloWorld</value></data></array></value>"
+        recv_data = (header + body).encode('utf-8')
+        mock_connected_socket.recv.return_value = recv_data
+
+        expected_response = "HelloWorld"
+        response = hfx_filename.db_get_by_uuid(0, "pool_patch", "22345")
+        self.assertEqual(expected_response, response)
+
+    def test_read_field(self, mock_socket):
+        mock_connected_socket = MagicMock()
+        mock_socket.return_value = mock_connected_socket
+
+        header = "HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n"
+        body = "<value><array><data><value>success</value><value>file_name</value></data></array></value>"
+        recv_data = (header + body).encode('utf-8')
+        mock_connected_socket.recv.return_value = recv_data
+
+        expected_filename = "file_name"
+        filename = hfx_filename.read_field(0, "pool_patch", "filename", "rf")
+        self.assertEqual(expected_filename, filename)
+
+
+@unittest.skipIf(sys.version_info < (3, 0), reason="requires python3")
+class TestParse(unittest.TestCase):
+
+    def test_parse_string(self): 
+        txt = "<value><array><data><value>success</value><value>abcde</value></data></array></value>" 
+        expected_txt = "abcde"
+        return_txt = hfx_filename.parse_string(txt)
+        self.assertEqual(expected_txt, return_txt)
+
+

scripts/test_perfmon.py

@@ -5,35 +5,13 @@
 import unittest
 from mock import MagicMock, patch
 import sys
-import os
-import subprocess
 import math
+from import_file import get_module
 
 # mock modules to avoid dependencies
 sys.modules["XenAPI"] = MagicMock()
 
-def import_from_file(module_name, file_path):
-    """Import a file as a module"""
-    if sys.version_info.major == 2:
-        return None
-    else:
-        from importlib import machinery, util
-        loader = machinery.SourceFileLoader(module_name, file_path)
-        spec = util.spec_from_loader(module_name, loader)
-        assert spec
-        assert spec.loader
-        module = util.module_from_spec(spec)
-        # Probably a good idea to add manually imported module stored in sys.modules
-        sys.modules[module_name] = module
-        spec.loader.exec_module(module)
-        return module
-
-def get_module():
-    """Import the perfmon script as a module for executing unit tests on functions"""
-    testdir = os.path.dirname(__file__)
-    return import_from_file("perfmon", testdir + "/perfmon")
-
-perfmon = get_module()
+perfmon = get_module("perfmon", "/perfmon")
 @unittest.skipIf(sys.version_info < (3, 0), reason="requires python3")
 @patch("subprocess.getoutput")
 class TestGetPercentage(unittest.TestCase):

scripts/test_static_vdis.py

@@ -5,36 +5,14 @@
 import unittest
 from mock import MagicMock
 import sys
-import os
-import subprocess
 import tempfile
+from import_file import get_module
 
 # mock modules to avoid dependencies
 sys.modules["XenAPI"] = MagicMock()
 sys.modules["inventory"] = MagicMock()
 
-def import_from_file(module_name, file_path):
-    """Import a file as a module"""
-    if sys.version_info.major == 2:
-        return None
-    else:
-        from importlib import machinery, util
-        loader = machinery.SourceFileLoader(module_name, file_path)
-        spec = util.spec_from_loader(module_name, loader)
-        assert spec
-        assert spec.loader
-        module = util.module_from_spec(spec)
-        # Probably a good idea to add manually imported module stored in sys.modules
-        sys.modules[module_name] = module
-        spec.loader.exec_module(module)
-        return module
-
-def get_module():
-    """Import the static-vdis script as a module for executing unit tests on functions"""
-    testdir = os.path.dirname(__file__)
-    return import_from_file("static_vdis", testdir + "/static-vdis")
-
-static_vdis = get_module()
+static_vdis = get_module("static_vdis", "/static-vdis")
 
 @unittest.skipIf(sys.version_info < (3, 0), reason="requires python3")     
 class TestReadWriteFile(unittest.TestCase):