Execute functional test jobs running under python3

Functional tests are executed by trovestack script, which uses python
in default OS version which is 2.7.

This change wraps python in tox and executes trovestack int_tests in
virtual enviroment, starting them in python3.

Any future python version change could be managed by tox framework.

Change-Id: I3a849978241d3d0669ef9e1c802ff504ac3c32cb
Signed-off-by: Marcin Piwowarczyk <m.piwowarczy@samsung.com>

integration/tests/integration/tests/util/services.py

@@ -189,12 +189,12 @@ class Service(object):
         # as python /path/to/executable args, so the entry is
         # /path/to/executable
         actual_command = self.cmd[proc_name_index].split("/")[-1]
-        print actual_command
+        print(actual_command)
         proc_command = ["/usr/bin/pgrep", "-f", actual_command]
-        print proc_command
+        print(proc_command)
         proc = start_proc(proc_command, shell=False)
         line = proc.stdout.readline()
-        print line
+        print(line)
         # pgrep only returns a pid. if there is no pid, it'll return nothing
         return len(line) != 0
 

roles/trove-devstack/tasks/main.yml

@@ -5,5 +5,5 @@
     export PATH_DEVSTACK_SRC=$DEST/devstack
     export TROVE_RESIZE_TIME_OUT={{trove_resize_time_out}}
 
-    cd $DEST/trove/integration/scripts
+    cd $DEST/trove
-    ./trovestack gate-tests {{trove_test_datastore}} {{trove_test_group}}
+    tox -etrovestack -vv -- gate-tests {{trove_test_datastore}} {{trove_test_group}}

tox.ini

@@ -123,3 +123,14 @@ deps =
   -c{toxinidir}/lower-constraints.txt
   -r{toxinidir}/test-requirements.txt
   -r{toxinidir}/requirements.txt
+
+[testenv:trovestack]
+basepython = python3
+skip_install = True
+changedir = {toxinidir}/integration/scripts
+passenv = *
+commands =
+    pip install --no-binary :all: {toxinidir} \
+      -c/opt/stack/trove/test-upper-constraints.txt \
+      -chttps://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt
+    ./trovestack {posargs}

trove/common/utils.py

@@ -185,14 +185,14 @@ class MethodInspector(object):
 
 
 def build_polling_task(retriever, condition=lambda value: value,
-                       sleep_time=1, time_out=None):
+                       sleep_time=1, time_out=0):
     start_time = time.time()
 
     def poll_and_check():
         obj = retriever()
         if condition(obj):
             raise loopingcall.LoopingCallDone(retvalue=obj)
-        if time_out is not None and time.time() - start_time > time_out:
+        if time_out > 0 and time.time() - start_time > time_out:
             raise exception.PollTimeOut
 
     return loopingcall.BackOffLoopingCall(
@@ -202,7 +202,7 @@ def build_polling_task(retriever, condition=lambda value: value,
 
 
 def poll_until(retriever, condition=lambda value: value,
-               sleep_time=1, time_out=None):
+               sleep_time=1, time_out=0):
     """Retrieves object until it passes condition, then returns it.
 
     If time_out_limit is passed in, PollTimeOut will be raised once that

trove/tests/api/configurations.py

@@ -138,7 +138,7 @@ def _test_configuration_is_applied_to_instance(instance, configuration_id):
         resp, body = instance_info.dbaas.client.last_response
         print(resp)
         print(body)
-        return json.loads(body)['type']
+        return json.loads(body.decode())['type']
 
     # check the config values are correct
     for key, value in actual_values:

trove/tests/api/replication.py

@@ -53,7 +53,7 @@ def _get_user_count(server_info):
            ' where user like \\\"slave_%\\\"\\\'')
     server = create_server_connection(server_info.id)
     stdout, stderr = server.execute(cmd)
-    return int(stdout.rstrip())
+    return int(stdout)
 
 
 def slave_is_running(running=True):
@@ -364,10 +364,7 @@ class DetachReplica(object):
             cmd = "mysql -BNq -e \\\'select @@read_only\\\'"
             server = create_server_connection(slave_instance.id)
             stdout, stderr = server.execute(cmd)
-            if (stdout.rstrip() != "0"):
+            return stdout.rstrip() == "0"
-                return False
-            else:
-                return True
         poll_until(check_not_read_only)
 
 

trove/tests/scenario/helpers/mysql_helper.py

@@ -20,7 +20,7 @@ class MysqlHelper(SqlHelper):
 
     def __init__(self, expected_override_name, report):
         super(MysqlHelper, self).__init__(expected_override_name, report,
-                                          'mysql')
+                                          'mysql+pymysql')
 
     def get_helper_credentials(self):
         return {'name': 'lite', 'password': 'litepass', 'database': 'firstdb'}

trove/tests/scenario/helpers/sql_helper.py

@@ -27,7 +27,8 @@ class SqlHelper(TestHelper):
 
     DATA_COLUMN_NAME = 'value'
 
-    def __init__(self, expected_override_name, report, protocol, port=None):
+    def __init__(self, expected_override_name, report,
+                 protocol="mysql+pymysql", port=None):
         super(SqlHelper, self).__init__(expected_override_name, report)
 
         self.protocol = protocol

trove/tests/scenario/runners/configuration_runners.py

@@ -144,7 +144,7 @@ class ConfigurationRunner(TestRunner):
             configuration.has_field('description', six.string_types)
             configuration.has_field('values', dict)
             configuration.has_field('datastore_name', six.string_types)
-            configuration.has_field('datastore_version_id', unicode)
+            configuration.has_field('datastore_version_id', six.text_type)
             configuration.has_field('datastore_version_name', six.string_types)
 
             self.assert_equal(name, result.name)

trove/tests/scenario/runners/module_runners.py

@@ -17,6 +17,7 @@
 import os
 from proboscis import SkipTest
 import re
+import six
 import tempfile
 import time
 
@@ -43,7 +44,7 @@ class ModuleRunner(TestRunner):
         self.MODULE_BINARY_SUFFIX = '_bin_auto'
         self.MODULE_BINARY_SUFFIX2 = self.MODULE_BINARY_SUFFIX + '_2'
         self.MODULE_BINARY_CONTENTS = os.urandom(20)
-        self.MODULE_BINARY_CONTENTS2 = '\x00\xFF\xea\x9c\x11\xfeok\xb1\x8ax'
+        self.MODULE_BINARY_CONTENTS2 = b'\x00\xFF\xea\x9c\x11\xfeok\xb1\x8ax'
 
         self.module_name_order = [
             {'suffix': self.MODULE_BINARY_SUFFIX,
@@ -1283,12 +1284,12 @@ class ModuleRunner(TestRunner):
                     if 'contents' in expected and expected['contents']:
                         with open(filename, 'rb') as fh:
                             contents = fh.read()
-                        # convert contents into bytearray to work with py27
+
-                        # and py34
+                        expected = expected['contents']
-                        contents = bytes([ord(item) for item in contents])
+                        if isinstance(expected, six.string_types):
-                        expected_contents = bytes(
+                            expected = expected.encode()
-                            [ord(item) for item in expected['contents']])
+
-                        self.assert_equal(expected_contents, contents,
+                        self.assert_equal(expected, contents,
                                           "Unexpected contents for %s" %
                                           module_name)
         finally:

trove/tests/scenario/runners/test_runners.py

@@ -145,7 +145,7 @@ class RunnerFactory(object):
                 # Only fail silently if it's something we expect,
                 # such as a missing override class.  Anything else
                 # shouldn't be suppressed.
-                l_msg = ie.message.lower()
+                l_msg = str(ie).lower()
                 if (load_type and load_type not in l_msg) or (
                         'no module named' not in l_msg and
                         'cannot be found' not in l_msg):
@@ -401,7 +401,16 @@ class TestRunner(object):
         """Assert that two lists contain same elements
         (with same multiplicities) ignoring the element order.
         """
-        return cls.assert_equal(sorted(expected), sorted(actual), message)
+        # Sorts the elements of a given list, including dictionaries.
+        # For dictionaries sorts based on dictionary key.
+        # example:
+        # [1, 3, 2]             -> [1, 2, 3]
+        # ["b", "a", "c"]       -> ["a", "b", "c"]
+        # [{'b':'y'},{'a':'x'}] -> [{'a':'x'},{'b':'y'}]
+        sort = lambda object: sorted(object, key=lambda e: sorted(e.keys())
+                                     if isinstance(e, dict) else e)
+
+        return cls.assert_equal(sort(expected), sort(actual), message)
 
     @classmethod
     def assert_equal(cls, expected, actual, message=None):
@@ -506,7 +515,7 @@ class TestRunner(object):
         if client:
             # Make sure that the client_cmd comes from the same client that
             # was passed in, otherwise asserting the client code may fail.
-            cmd_clz = client_cmd.im_self
+            cmd_clz = client_cmd.__self__
             cmd_clz_name = cmd_clz.__class__.__name__
             client_attrs = [attr[0] for attr in inspect.getmembers(
                             client.real_client)
@@ -700,7 +709,7 @@ class TestRunner(object):
         return False
 
     def _poll_while(self, instance_id, expected_status,
-                    sleep_time=1, time_out=None):
+                    sleep_time=1, time_out=0):
         poll_until(lambda: not self._has_status(instance_id, expected_status),
                    sleep_time=sleep_time, time_out=time_out)
 

trove/tests/util/event_simulator.py

@@ -227,7 +227,7 @@ def fake_sleep(time_to_sleep):
 
 
 def fake_poll_until(retriever, condition=lambda value: value,
-                    sleep_time=1, time_out=None):
+                    sleep_time=1, time_out=0):
     """Fakes out poll until."""
     from trove.common import exception
     slept_time = 0

trove/tests/util/server_connection.py

@@ -48,7 +48,8 @@ class ServerSSHConnection(object):
     def execute(self, cmd):
         exe_cmd = "%s %s %s" % (tests.SSH_CMD, self.ip_address, cmd)
         print("RUNNING COMMAND: %s" % exe_cmd)
-        return util.process(exe_cmd)
+        stdout, stderr = util.process(exe_cmd)
+        return (stdout.decode(), stderr.decode())
 
 
 class OpenVZServerConnection(object):