From e9cc6ca372e627673113cc36d582732968b74600 Mon Sep 17 00:00:00 2001 From: Marcin Piwowarczyk Date: Mon, 24 Sep 2018 17:02:24 +0200 Subject: [PATCH] Execute functional test jobs running under python3 Functional tests are executed by trovestack script, which uses python in default OS version which is 2.7. This change wraps python in tox and executes trovestack int_tests in virtual enviroment, starting them in python3. Any future python version change could be managed by tox framework. Change-Id: I3a849978241d3d0669ef9e1c802ff504ac3c32cb Signed-off-by: Marcin Piwowarczyk --- .../tests/integration/tests/util/services.py | 6 +++--- roles/trove-devstack/tasks/main.yml | 4 ++-- tox.ini | 11 +++++++++++ trove/common/utils.py | 6 +++--- trove/tests/api/configurations.py | 2 +- trove/tests/api/replication.py | 7 ++----- trove/tests/scenario/helpers/mysql_helper.py | 2 +- trove/tests/scenario/helpers/sql_helper.py | 3 ++- .../scenario/runners/configuration_runners.py | 2 +- trove/tests/scenario/runners/module_runners.py | 15 ++++++++------- trove/tests/scenario/runners/test_runners.py | 17 +++++++++++++---- trove/tests/util/event_simulator.py | 2 +- trove/tests/util/server_connection.py | 3 ++- 13 files changed, 50 insertions(+), 30 deletions(-) diff --git a/integration/tests/integration/tests/util/services.py b/integration/tests/integration/tests/util/services.py index f5f8cbf1cb..6e8f1da88c 100644 --- a/integration/tests/integration/tests/util/services.py +++ b/integration/tests/integration/tests/util/services.py @@ -189,12 +189,12 @@ class Service(object): # as python /path/to/executable args, so the entry is # /path/to/executable actual_command = self.cmd[proc_name_index].split("/")[-1] - print actual_command + print(actual_command) proc_command = ["/usr/bin/pgrep", "-f", actual_command] - print proc_command + print(proc_command) proc = start_proc(proc_command, shell=False) line = proc.stdout.readline() - print line + print(line) # pgrep only returns a pid. if there is no pid, it'll return nothing return len(line) != 0 diff --git a/roles/trove-devstack/tasks/main.yml b/roles/trove-devstack/tasks/main.yml index 7ee91646b1..01132837f9 100644 --- a/roles/trove-devstack/tasks/main.yml +++ b/roles/trove-devstack/tasks/main.yml @@ -5,5 +5,5 @@ export PATH_DEVSTACK_SRC=$DEST/devstack export TROVE_RESIZE_TIME_OUT={{trove_resize_time_out}} - cd $DEST/trove/integration/scripts - ./trovestack gate-tests {{trove_test_datastore}} {{trove_test_group}} + cd $DEST/trove + tox -etrovestack -vv -- gate-tests {{trove_test_datastore}} {{trove_test_group}} diff --git a/tox.ini b/tox.ini index 5a7d7caf61..1ced348d8f 100644 --- a/tox.ini +++ b/tox.ini @@ -123,3 +123,14 @@ deps = -c{toxinidir}/lower-constraints.txt -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt + +[testenv:trovestack] +basepython = python3 +skip_install = True +changedir = {toxinidir}/integration/scripts +passenv = * +commands = + pip install --no-binary :all: {toxinidir} \ + -c/opt/stack/trove/test-upper-constraints.txt \ + -chttps://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt + ./trovestack {posargs} \ No newline at end of file diff --git a/trove/common/utils.py b/trove/common/utils.py index 484c044e7f..7710923c38 100644 --- a/trove/common/utils.py +++ b/trove/common/utils.py @@ -185,14 +185,14 @@ class MethodInspector(object): def build_polling_task(retriever, condition=lambda value: value, - sleep_time=1, time_out=None): + sleep_time=1, time_out=0): start_time = time.time() def poll_and_check(): obj = retriever() if condition(obj): raise loopingcall.LoopingCallDone(retvalue=obj) - if time_out is not None and time.time() - start_time > time_out: + if time_out > 0 and time.time() - start_time > time_out: raise exception.PollTimeOut return loopingcall.BackOffLoopingCall( @@ -202,7 +202,7 @@ def build_polling_task(retriever, condition=lambda value: value, def poll_until(retriever, condition=lambda value: value, - sleep_time=1, time_out=None): + sleep_time=1, time_out=0): """Retrieves object until it passes condition, then returns it. If time_out_limit is passed in, PollTimeOut will be raised once that diff --git a/trove/tests/api/configurations.py b/trove/tests/api/configurations.py index 1a7dbaa8ed..c2dc968c92 100644 --- a/trove/tests/api/configurations.py +++ b/trove/tests/api/configurations.py @@ -138,7 +138,7 @@ def _test_configuration_is_applied_to_instance(instance, configuration_id): resp, body = instance_info.dbaas.client.last_response print(resp) print(body) - return json.loads(body)['type'] + return json.loads(body.decode())['type'] # check the config values are correct for key, value in actual_values: diff --git a/trove/tests/api/replication.py b/trove/tests/api/replication.py index 57b0662c93..9a16a2ec4a 100644 --- a/trove/tests/api/replication.py +++ b/trove/tests/api/replication.py @@ -53,7 +53,7 @@ def _get_user_count(server_info): ' where user like \\\"slave_%\\\"\\\'') server = create_server_connection(server_info.id) stdout, stderr = server.execute(cmd) - return int(stdout.rstrip()) + return int(stdout) def slave_is_running(running=True): @@ -364,10 +364,7 @@ class DetachReplica(object): cmd = "mysql -BNq -e \\\'select @@read_only\\\'" server = create_server_connection(slave_instance.id) stdout, stderr = server.execute(cmd) - if (stdout.rstrip() != "0"): - return False - else: - return True + return stdout.rstrip() == "0" poll_until(check_not_read_only) diff --git a/trove/tests/scenario/helpers/mysql_helper.py b/trove/tests/scenario/helpers/mysql_helper.py index 7fada6817b..60f202c6d7 100644 --- a/trove/tests/scenario/helpers/mysql_helper.py +++ b/trove/tests/scenario/helpers/mysql_helper.py @@ -20,7 +20,7 @@ class MysqlHelper(SqlHelper): def __init__(self, expected_override_name, report): super(MysqlHelper, self).__init__(expected_override_name, report, - 'mysql') + 'mysql+pymysql') def get_helper_credentials(self): return {'name': 'lite', 'password': 'litepass', 'database': 'firstdb'} diff --git a/trove/tests/scenario/helpers/sql_helper.py b/trove/tests/scenario/helpers/sql_helper.py index 325fb03e77..a158d963aa 100644 --- a/trove/tests/scenario/helpers/sql_helper.py +++ b/trove/tests/scenario/helpers/sql_helper.py @@ -27,7 +27,8 @@ class SqlHelper(TestHelper): DATA_COLUMN_NAME = 'value' - def __init__(self, expected_override_name, report, protocol, port=None): + def __init__(self, expected_override_name, report, + protocol="mysql+pymysql", port=None): super(SqlHelper, self).__init__(expected_override_name, report) self.protocol = protocol diff --git a/trove/tests/scenario/runners/configuration_runners.py b/trove/tests/scenario/runners/configuration_runners.py index 40fba0689d..a2c9c5f5c2 100644 --- a/trove/tests/scenario/runners/configuration_runners.py +++ b/trove/tests/scenario/runners/configuration_runners.py @@ -144,7 +144,7 @@ class ConfigurationRunner(TestRunner): configuration.has_field('description', six.string_types) configuration.has_field('values', dict) configuration.has_field('datastore_name', six.string_types) - configuration.has_field('datastore_version_id', unicode) + configuration.has_field('datastore_version_id', six.text_type) configuration.has_field('datastore_version_name', six.string_types) self.assert_equal(name, result.name) diff --git a/trove/tests/scenario/runners/module_runners.py b/trove/tests/scenario/runners/module_runners.py index c6483e31a8..e78065162d 100644 --- a/trove/tests/scenario/runners/module_runners.py +++ b/trove/tests/scenario/runners/module_runners.py @@ -17,6 +17,7 @@ import os from proboscis import SkipTest import re +import six import tempfile import time @@ -43,7 +44,7 @@ class ModuleRunner(TestRunner): self.MODULE_BINARY_SUFFIX = '_bin_auto' self.MODULE_BINARY_SUFFIX2 = self.MODULE_BINARY_SUFFIX + '_2' self.MODULE_BINARY_CONTENTS = os.urandom(20) - self.MODULE_BINARY_CONTENTS2 = '\x00\xFF\xea\x9c\x11\xfeok\xb1\x8ax' + self.MODULE_BINARY_CONTENTS2 = b'\x00\xFF\xea\x9c\x11\xfeok\xb1\x8ax' self.module_name_order = [ {'suffix': self.MODULE_BINARY_SUFFIX, @@ -1283,12 +1284,12 @@ class ModuleRunner(TestRunner): if 'contents' in expected and expected['contents']: with open(filename, 'rb') as fh: contents = fh.read() - # convert contents into bytearray to work with py27 - # and py34 - contents = bytes([ord(item) for item in contents]) - expected_contents = bytes( - [ord(item) for item in expected['contents']]) - self.assert_equal(expected_contents, contents, + + expected = expected['contents'] + if isinstance(expected, six.string_types): + expected = expected.encode() + + self.assert_equal(expected, contents, "Unexpected contents for %s" % module_name) finally: diff --git a/trove/tests/scenario/runners/test_runners.py b/trove/tests/scenario/runners/test_runners.py index 58e079a640..43a4fecddf 100644 --- a/trove/tests/scenario/runners/test_runners.py +++ b/trove/tests/scenario/runners/test_runners.py @@ -145,7 +145,7 @@ class RunnerFactory(object): # Only fail silently if it's something we expect, # such as a missing override class. Anything else # shouldn't be suppressed. - l_msg = ie.message.lower() + l_msg = str(ie).lower() if (load_type and load_type not in l_msg) or ( 'no module named' not in l_msg and 'cannot be found' not in l_msg): @@ -401,7 +401,16 @@ class TestRunner(object): """Assert that two lists contain same elements (with same multiplicities) ignoring the element order. """ - return cls.assert_equal(sorted(expected), sorted(actual), message) + # Sorts the elements of a given list, including dictionaries. + # For dictionaries sorts based on dictionary key. + # example: + # [1, 3, 2] -> [1, 2, 3] + # ["b", "a", "c"] -> ["a", "b", "c"] + # [{'b':'y'},{'a':'x'}] -> [{'a':'x'},{'b':'y'}] + sort = lambda object: sorted(object, key=lambda e: sorted(e.keys()) + if isinstance(e, dict) else e) + + return cls.assert_equal(sort(expected), sort(actual), message) @classmethod def assert_equal(cls, expected, actual, message=None): @@ -506,7 +515,7 @@ class TestRunner(object): if client: # Make sure that the client_cmd comes from the same client that # was passed in, otherwise asserting the client code may fail. - cmd_clz = client_cmd.im_self + cmd_clz = client_cmd.__self__ cmd_clz_name = cmd_clz.__class__.__name__ client_attrs = [attr[0] for attr in inspect.getmembers( client.real_client) @@ -700,7 +709,7 @@ class TestRunner(object): return False def _poll_while(self, instance_id, expected_status, - sleep_time=1, time_out=None): + sleep_time=1, time_out=0): poll_until(lambda: not self._has_status(instance_id, expected_status), sleep_time=sleep_time, time_out=time_out) diff --git a/trove/tests/util/event_simulator.py b/trove/tests/util/event_simulator.py index d21254de04..79f5a4ba36 100644 --- a/trove/tests/util/event_simulator.py +++ b/trove/tests/util/event_simulator.py @@ -227,7 +227,7 @@ def fake_sleep(time_to_sleep): def fake_poll_until(retriever, condition=lambda value: value, - sleep_time=1, time_out=None): + sleep_time=1, time_out=0): """Fakes out poll until.""" from trove.common import exception slept_time = 0 diff --git a/trove/tests/util/server_connection.py b/trove/tests/util/server_connection.py index 6eed36633a..156b8c9865 100644 --- a/trove/tests/util/server_connection.py +++ b/trove/tests/util/server_connection.py @@ -48,7 +48,8 @@ class ServerSSHConnection(object): def execute(self, cmd): exe_cmd = "%s %s %s" % (tests.SSH_CMD, self.ip_address, cmd) print("RUNNING COMMAND: %s" % exe_cmd) - return util.process(exe_cmd) + stdout, stderr = util.process(exe_cmd) + return (stdout.decode(), stderr.decode()) class OpenVZServerConnection(object):