From ff9c4ac1513a8d13cea376ab7ff99c1d5cb3fc72 Mon Sep 17 00:00:00 2001 From: Andrey Kurilin Date: Mon, 20 Oct 2014 17:11:37 +0300 Subject: [PATCH] Refactor tempest verification and tests for it Class `verification.verifiers.tempest.Tempest` contains next changes: Output from installation of virtual environment is not useful for users, so visibility of this output was changed and now only in debug mode it will be printed. The status of verification will be changed to 'FAILED' if something will go wrong in subproccess.call Two variables contain word 'tempest' in their names. This is redundant, so they were renamed: Tempest.tempest_base_path => Tempest.base_repo Tempest.tempest_path => Tempest._path Construction "os.path.join(Tempest.path, some_path)" was moved to method `Tempest.path`, since it used in many places. Method `Tempest.parse_results` should not be static, because it needed inner variable `Tempest.log_file_raw`, so this was fixed. "git remote update" is not needed in Tempest installation, so we can remove this call and decrease time of installation. In `rally.cmd.commands.verify.start` command, several issues were fixed: First function argument changed to "set_name" instead of "deploy_id". Reason: "deploy_id" have default value, so it should be the first in arguments. It will simplify command for end-users(launch 'rally verify start ' instead of 'rally verify start --set '). Task commands have cool feature: save task_id in global variables, so results cmd can print the last task, without setting it id. This feature is ported in verification. Tests for verification contains a lot of tests, so they are splitted to separate classes(TempestVerifyTestCase, TempestInstallAndUninstallTestCase and etc). Also, new tests were added. Change-Id: I08a52a1e3ceb468ba619049573bcfe642aecbcaf --- rally/benchmark/context/tempest.py | 4 +- rally/cmd/cliutils.py | 2 +- rally/cmd/commands/use.py | 13 + rally/cmd/commands/verify.py | 27 +- rally/cmd/envutils.py | 9 +- rally/orchestrator/api.py | 2 + .../verification/verifiers/tempest/tempest.py | 96 ++--- tests/unit/benchmark/context/test_tempest.py | 4 +- .../scenarios/tempest/test_tempest.py | 22 +- tests/unit/cmd/commands/test_verify.py | 9 +- .../verification/verifiers/test_tempest.py | 354 ++++++++++++------ tools/rally.bash_completion | 3 +- 12 files changed, 347 insertions(+), 198 deletions(-) diff --git a/rally/benchmark/context/tempest.py b/rally/benchmark/context/tempest.py index 5c129b1d30..c36645d085 100644 --- a/rally/benchmark/context/tempest.py +++ b/rally/benchmark/context/tempest.py @@ -66,12 +66,12 @@ class Tempest(base.Context): cmd = ("cd %(tempest_dir)s " "&& %(venv)s python tempest/stress/tools/cleanup.py" % { - "tempest_dir": self.verifier.tempest_path, + "tempest_dir": self.verifier.path, "venv": self.verifier.venv_wrapper}) LOG.debug("Cleanup started by the command: %s" % cmd) subprocess.check_call(cmd, shell=True, env=self.verifier.env, - cwd=self.verifier.tempest_path) + cwd=self.verifier.path) except subprocess.CalledProcessError: LOG.error("Tempest cleanup failed.") diff --git a/rally/cmd/cliutils.py b/rally/cmd/cliutils.py index 9081f0d363..4d8efad83a 100644 --- a/rally/cmd/cliutils.py +++ b/rally/cmd/cliutils.py @@ -38,7 +38,7 @@ class CategoryParser(argparse.ArgumentParser): """Customized arguments parser We need this one to override hardcoded behavior. - So, we want to print item's help instead of 'error: too fiew arguments'. + So, we want to print item's help instead of 'error: too few arguments'. Also, we want not to print positional arguments in help messge. """ diff --git a/rally/cmd/commands/use.py b/rally/cmd/commands/use.py index aaa79bafd3..6458d902d2 100644 --- a/rally/cmd/commands/use.py +++ b/rally/cmd/commands/use.py @@ -113,3 +113,16 @@ class UseCommands(object): self._ensure_rally_configuration_dir_exists() db.task_get(task_id) self._update_attribute_in_global_file('RALLY_TASK', task_id) + + @cliutils.args("--uuid", type=str, dest="verification_id", required=False, + help="UUID of the verification") + def verification(self, verification_id): + """Set active verification. + + :param verification_id: a UUID of verification + """ + print('Verification UUID: %s' % verification_id) + self._ensure_rally_configuration_dir_exists() + db.verification_get(verification_id) + self._update_attribute_in_global_file('RALLY_VERIFICATION', + verification_id) diff --git a/rally/cmd/commands/verify.py b/rally/cmd/commands/verify.py index 5d5b96f706..6040ba04a2 100644 --- a/rally/cmd/commands/verify.py +++ b/rally/cmd/commands/verify.py @@ -22,6 +22,7 @@ import pprint import six from rally.cmd import cliutils +from rally.cmd.commands import use from rally.cmd import envutils from rally import consts from rally import db @@ -50,13 +51,15 @@ class VerifyCommands(object): @cliutils.args("--tempest-config", dest="tempest_config", type=str, required=False, help="User specified Tempest config file location") + @cliutils.args("--no-use", action="store_false", dest="do_use", + help="Don't set new task as default for future operations") @envutils.with_default_deploy_id - def start(self, deploy_id=None, set_name="smoke", regex=None, - tempest_config=None): + def start(self, set_name="smoke", deploy_id=None, regex=None, + tempest_config=None, do_use=False): """Start set of tests. - :param deploy_id: a UUID of a deployment :param set_name: Name of tempest test set + :param deploy_id: a UUID of a deployment :param regex: Regular expression of test :param tempest_config: User specified Tempest config file location """ @@ -66,9 +69,10 @@ class VerifyCommands(object): if set_name not in consts.TEMPEST_TEST_SETS: print("Sorry, but there are no desired tempest test set. Please " "choose from: %s" % ", ".join(consts.TEMPEST_TEST_SETS)) - return(1) - - api.verify(deploy_id, set_name, regex, tempest_config) + return (1) + verification = api.verify(deploy_id, set_name, regex, tempest_config) + if do_use: + use.UseCommands().verification(verification['uuid']) def list(self): """Display all verifications table, started and finished.""" @@ -94,8 +98,9 @@ class VerifyCommands(object): @cliutils.args('--output-file', type=str, required=False, dest='output_file', help='If specified, output will be saved to given file') - def results(self, verification_uuid, output_file=None, output_html=None, - output_json=None, output_pprint=None): + @envutils.with_default_verification_id + def results(self, verification_uuid=None, output_file=None, + output_html=None, output_json=None, output_pprint=None): """Get raw results of the verification. :param verification_uuid: Verification UUID @@ -139,7 +144,8 @@ class VerifyCommands(object): help='Tests can be sorted by "name" or "duration"') @cliutils.args('--detailed', dest='detailed', action='store_true', required=False, help='Prints traceback of failed tests') - def show(self, verification_uuid, sort_by='name', detailed=False): + @envutils.with_default_verification_id + def show(self, verification_uuid=None, sort_by='name', detailed=False): """Display results table of the verification.""" try: @@ -191,7 +197,8 @@ class VerifyCommands(object): help='UUID of a verification') @cliutils.args('--sort-by', dest='sort_by', type=str, required=False, help='Tests can be sorted by "name" or "duration"') - def detailed(self, verification_uuid, sort_by='name'): + @envutils.with_default_verification_id + def detailed(self, verification_uuid=None, sort_by='name'): """Display results table of verification with detailed errors.""" self.show(verification_uuid, sort_by, True) diff --git a/rally/cmd/envutils.py b/rally/cmd/envutils.py index b7125a773d..82ea7c568e 100644 --- a/rally/cmd/envutils.py +++ b/rally/cmd/envutils.py @@ -21,9 +21,10 @@ from rally import exceptions from rally import fileutils from rally.i18n import _ -ENV_DEPLOYMENT = 'RALLY_DEPLOYMENT' -ENV_TASK = 'RALLY_TASK' -ENVVARS = [ENV_DEPLOYMENT, ENV_TASK] +ENV_DEPLOYMENT = "RALLY_DEPLOYMENT" +ENV_TASK = "RALLY_TASK" +ENV_VERIFICATION = "RALLY_VERIFICATION" +ENVVARS = [ENV_DEPLOYMENT, ENV_TASK, ENV_VERIFICATION] MSG_MISSING_ARG = _("Missing argument: --%(arg_name)s") @@ -68,3 +69,5 @@ with_default_deploy_id = default_from_global('deploy_id', ENV_DEPLOYMENT, "uuid") with_default_task_id = default_from_global('task_id', ENV_TASK, "uuid") +with_default_verification_id = default_from_global( + "verification_uuid", ENV_VERIFICATION, "uuid") diff --git a/rally/orchestrator/api.py b/rally/orchestrator/api.py index 5349b499de..9bda37aa39 100644 --- a/rally/orchestrator/api.py +++ b/rally/orchestrator/api.py @@ -177,3 +177,5 @@ def verify(deploy_id, set_name, regex, tempest_config): verification.set_running() verifier.verify(set_name=set_name, regex=regex) + + return verification diff --git a/rally/verification/verifiers/tempest/tempest.py b/rally/verification/verifiers/tempest/tempest.py index 63bea3b81f..34c03864cf 100644 --- a/rally/verification/verifiers/tempest/tempest.py +++ b/rally/verification/verifiers/tempest/tempest.py @@ -35,30 +35,33 @@ class TempestSetupFailure(exceptions.RallyException): msg_fmt = _("Unable to setup tempest: '%(message)s'") +def check_output(*args, **kwargs): + output = subprocess.check_output(*args, **kwargs) + + if LOG.getEffectiveLevel() <= logging.DEBUG: + print(output) + + class Tempest(object): - tempest_base_path = os.path.join(os.path.expanduser("~"), - ".rally/tempest/base") + base_repo = os.path.join(os.path.expanduser("~"), ".rally/tempest/base") def __init__(self, deploy_id, verification=None, tempest_config=None): self.deploy_id = deploy_id - self.tempest_path = os.path.join(os.path.expanduser("~"), - ".rally/tempest", - "for-deployment-%s" % deploy_id) - self.config_file = tempest_config or os.path.join(self.tempest_path, - "tempest.conf") - self.log_file_raw = os.path.join(self.tempest_path, "subunit.stream") - self.venv_wrapper = os.path.join(self.tempest_path, - "tools/with_venv.sh") + self._path = os.path.join(os.path.expanduser("~"), + ".rally/tempest", + "for-deployment-%s" % deploy_id) + self.config_file = tempest_config or self.path("tempest.conf") + self.log_file_raw = self.path("subunit.stream") + self.venv_wrapper = self.path("tools/with_venv.sh") self.verification = verification self._env = None def _generate_env(self): env = os.environ.copy() - env["TEMPEST_CONFIG_DIR"] = os.path.split(self.config_file)[0] + env["TEMPEST_CONFIG_DIR"] = os.path.dirname(self.config_file) env["TEMPEST_CONFIG"] = os.path.basename(self.config_file) - env["OS_TEST_PATH"] = os.path.join(self.tempest_path, - "tempest/test_discover") + env["OS_TEST_PATH"] = self.path("tempest/test_discover") LOG.debug("Generated environ: %s" % env) self._env = env @@ -68,18 +71,22 @@ class Tempest(object): self._generate_env() return self._env + def path(self, *inner_path): + if inner_path: + return os.path.join(self._path, *inner_path) + return self._path + def _install_venv(self): - if not os.path.isdir(os.path.join(self.tempest_path, '.venv')): - LOG.info('Validating python environment') + path_to_venv = self.path(".venv") + + if not os.path.isdir(path_to_venv): self.validate_env() - LOG.info("No virtual environment found...Install the virtualenv.") - LOG.debug("Virtual environment directory: %s" % - os.path.join(self.tempest_path, ".venv")) - subprocess.check_call("python ./tools/install_venv.py", shell=True, - cwd=self.tempest_path) - subprocess.check_call( - "%s python setup.py install" % self.venv_wrapper, - shell=True, cwd=self.tempest_path) + print("No virtual environment found...Install the virtualenv.") + LOG.debug("Virtual environment directory: %s" % path_to_venv) + check_output("python ./tools/install_venv.py", shell=True, + cwd=self.path()) + check_output("%s python setup.py install" % self.venv_wrapper, + shell=True, cwd=self.path()) def is_configured(self): return os.path.isfile(self.config_file) @@ -98,16 +105,15 @@ class Tempest(object): LOG.info("Tempest is already configured.") def _initialize_testr(self): - if not os.path.isdir(os.path.join(self.tempest_path, - ".testrepository")): + if not os.path.isdir(self.path(".testrepository")): msg = _("Test Repository initialization.") LOG.info(_("Starting: ") + msg) subprocess.check_call("%s testr init" % self.venv_wrapper, - shell=True, cwd=self.tempest_path) + shell=True, cwd=self.path()) LOG.info(_("Completed: ") + msg) def is_installed(self): - return os.path.exists(os.path.join(self.tempest_path, ".venv")) + return os.path.exists(self.path(".venv")) @staticmethod def _clone(): @@ -115,22 +121,19 @@ class Tempest(object): "This could take a few minutes...") subprocess.check_call(["git", "clone", "https://github.com/openstack/tempest", - Tempest.tempest_base_path]) + Tempest.base_repo]) def install(self): if not self.is_installed(): try: - if not os.path.exists(Tempest.tempest_base_path): + if not os.path.exists(Tempest.base_repo): Tempest._clone() - if not os.path.exists(self.tempest_path): - shutil.copytree(Tempest.tempest_base_path, - self.tempest_path) + if not os.path.exists(self.path()): + shutil.copytree(Tempest.base_repo, self.path()) subprocess.check_call("git checkout master; " - "git remote update; " "git pull", shell=True, - cwd=os.path.join(self.tempest_path, - "tempest")) + cwd=self.path("tempest")) self._install_venv() self._initialize_testr() except subprocess.CalledProcessError as e: @@ -143,8 +146,8 @@ class Tempest(object): print("Tempest is already installed") def uninstall(self): - if os.path.exists(self.tempest_path): - shutil.rmtree(self.tempest_path) + if os.path.exists(self.path()): + shutil.rmtree(self.path()) @utils.log_verification_wrapper(LOG.info, _("Run verification.")) def _prepare_and_run(self, set_name, regex): @@ -167,7 +170,7 @@ class Tempest(object): try: self.run(testr_arg) except subprocess.CalledProcessError: - print("Test set %s has been finished with error. " + print("Test set '%s' has been finished with error. " "Check log for details" % set_name) def run(self, testr_arg=None, log_file=None, tempest_conf=None): @@ -197,11 +200,11 @@ class Tempest(object): { "venv": self.venv_wrapper, "arg": testr_arg, - "tempest_path": self.tempest_path, + "tempest_path": self.path(), "log_file": log_file or self.log_file_raw }) LOG.debug("Test(s) started by the command: %s" % test_cmd) - subprocess.check_call(test_cmd, cwd=self.tempest_path, + subprocess.check_call(test_cmd, cwd=self.path(), env=self.env, shell=True) def discover_tests(self, pattern=""): @@ -211,7 +214,7 @@ class Tempest(object): "venv": self.venv_wrapper, "pattern": pattern} raw_results = subprocess.Popen( - cmd, shell=True, cwd=self.tempest_path, env=self.env, + cmd, shell=True, cwd=self.path(), env=self.env, stdout=subprocess.PIPE).communicate()[0] tests = set() @@ -225,10 +228,9 @@ class Tempest(object): return tests - @staticmethod - def parse_results(log_file_raw): + def parse_results(self, log_file=None): """Parse subunit raw log file.""" - + log_file_raw = log_file or self.log_file_raw if os.path.isfile(log_file_raw): data = jsonutils.loads(subunit2json.main(log_file_raw)) return data['total'], data['test_cases'] @@ -239,10 +241,12 @@ class Tempest(object): @utils.log_verification_wrapper( LOG.info, _("Saving verification results.")) def _save_results(self): - total, test_cases = self.parse_results(self.log_file_raw) + total, test_cases = self.parse_results() if total and test_cases and self.verification: self.verification.finish_verification(total=total, test_cases=test_cases) + else: + self.verification.set_failed() def validate_env(self): """Validate environment parameters required for running tempest. @@ -252,7 +256,7 @@ class Tempest(object): if sys.version_info < (2, 7): raise exceptions.IncompatiblePythonVersion( - version=sys.version_info) + version=sys.version_info) def verify(self, set_name, regex): self._prepare_and_run(set_name, regex) diff --git a/tests/unit/benchmark/context/test_tempest.py b/tests/unit/benchmark/context/test_tempest.py index bfdaad1262..8031ba9ed9 100644 --- a/tests/unit/benchmark/context/test_tempest.py +++ b/tests/unit/benchmark/context/test_tempest.py @@ -102,8 +102,8 @@ class TempestContextTestCase(test.TestCase): mock_sp.check_call.assert_called_once_with( "cd %s && %s python tempest/stress/tools/cleanup.py" % - (benchmark.verifier.tempest_path, benchmark.verifier.venv_wrapper), - shell=True, cwd=benchmark.verifier.tempest_path, + (benchmark.verifier.path, benchmark.verifier.venv_wrapper), + shell=True, cwd=benchmark.verifier.path, env=benchmark.verifier.env) mock_shutil.rmtree.assert_called_once_with("/tmp/path") diff --git a/tests/unit/benchmark/scenarios/tempest/test_tempest.py b/tests/unit/benchmark/scenarios/tempest/test_tempest.py index ef9a0cf1b1..9a2dd9febc 100644 --- a/tests/unit/benchmark/scenarios/tempest/test_tempest.py +++ b/tests/unit/benchmark/scenarios/tempest/test_tempest.py @@ -44,7 +44,7 @@ class TempestScenarioTestCase(test.TestCase): "| %(venv)s %(tempest_path)s/tools/colorizer.py" % { "venv": self.verifier.venv_wrapper, - "tempest_path": self.verifier.tempest_path, + "tempest_path": self.verifier.path(), "tests": " ".join(tests) }) @@ -58,7 +58,7 @@ class TempestScenarioTestCase(test.TestCase): expected_call = self.get_tests_launcher_cmd([fake_test]) mock_sp.check_call.assert_called_once_with( - expected_call, cwd=self.verifier.tempest_path, + expected_call, cwd=self.verifier.path(), env=self.verifier.env, shell=True) @mock.patch(TS + ".utils.tempfile") @@ -71,7 +71,7 @@ class TempestScenarioTestCase(test.TestCase): expected_call = self.get_tests_launcher_cmd([fake_test]) mock_sp.check_call.assert_called_once_with( - expected_call, cwd=self.verifier.tempest_path, + expected_call, cwd=self.verifier.path(), env=self.verifier.env, shell=True) @mock.patch(TS + ".utils.tempfile") @@ -83,7 +83,7 @@ class TempestScenarioTestCase(test.TestCase): expected_call = self.get_tests_launcher_cmd(["tempest.api.network"]) mock_sp.check_call.assert_called_once_with( - expected_call, cwd=self.verifier.tempest_path, + expected_call, cwd=self.verifier.path(), env=self.verifier.env, shell=True) @mock.patch(TS + ".utils.tempfile") @@ -95,7 +95,7 @@ class TempestScenarioTestCase(test.TestCase): expected_call = self.get_tests_launcher_cmd([]) mock_sp.check_call.assert_called_once_with( - expected_call, cwd=self.verifier.tempest_path, + expected_call, cwd=self.verifier.path(), env=self.verifier.env, shell=True) @mock.patch(TS + ".utils.tempfile") @@ -107,7 +107,7 @@ class TempestScenarioTestCase(test.TestCase): expected_call = self.get_tests_launcher_cmd(["smoke"]) mock_sp.check_call.assert_called_once_with( - expected_call, cwd=self.verifier.tempest_path, + expected_call, cwd=self.verifier.path(), env=self.verifier.env, shell=True) @mock.patch(TS + ".utils.tempfile") @@ -119,7 +119,7 @@ class TempestScenarioTestCase(test.TestCase): expected_call = self.get_tests_launcher_cmd([]) mock_sp.check_call.assert_called_once_with( - expected_call, cwd=self.verifier.tempest_path, + expected_call, cwd=self.verifier.path(), env=self.verifier.env, shell=True) @mock.patch(TS + ".utils.tempfile") @@ -137,7 +137,7 @@ class TempestScenarioTestCase(test.TestCase): expected_call = self.get_tests_launcher_cmd([fake_test]) mock_sp.check_call.assert_called_once_with( - expected_call, cwd=self.verifier.tempest_path, + expected_call, cwd=self.verifier.path(), env=self.verifier.env, shell=True) @mock.patch(TS + ".utils.tempfile") @@ -149,7 +149,7 @@ class TempestScenarioTestCase(test.TestCase): expected_call = self.get_tests_launcher_cmd(["tempest.api.network"]) mock_sp.check_call.assert_called_once_with( - expected_call, cwd=self.verifier.tempest_path, + expected_call, cwd=self.verifier.path(), env=self.verifier.env, shell=True) @mock.patch(TS + ".utils.tempfile") @@ -162,7 +162,7 @@ class TempestScenarioTestCase(test.TestCase): expected_call = self.get_tests_launcher_cmd(fake_tests) mock_sp.check_call.assert_called_once_with( - expected_call, cwd=self.verifier.tempest_path, + expected_call, cwd=self.verifier.path(), env=self.verifier.env, shell=True) @mock.patch(TS + ".utils.tempfile") @@ -175,5 +175,5 @@ class TempestScenarioTestCase(test.TestCase): expected_call = self.get_tests_launcher_cmd([regex]) mock_sp.check_call.assert_called_once_with( - expected_call, cwd=self.verifier.tempest_path, + expected_call, cwd=self.verifier.path(), env=self.verifier.env, shell=True) diff --git a/tests/unit/cmd/commands/test_verify.py b/tests/unit/cmd/commands/test_verify.py index faa757fa2b..5496262a12 100644 --- a/tests/unit/cmd/commands/test_verify.py +++ b/tests/unit/cmd/commands/test_verify.py @@ -29,10 +29,6 @@ class VerifyCommandsTestCase(test.TestCase): def setUp(self): super(VerifyCommandsTestCase, self).setUp() self.verify = verify.VerifyCommands() - self.endpoint = {'endpoints': [{'auth_url': 'fake_auth_url', - 'username': 'fake_username', - 'password': 'fake_password', - 'tenant_name': 'fake_tenant_name'}]} self.image1 = mock.Mock() self.image1.name = 'cirros-1' @@ -57,7 +53,7 @@ class VerifyCommandsTestCase(test.TestCase): mock_clients().nova().flavors.list.return_value = [ self.flavor1, self.flavor2] - self.verify.start(deploy_id) + self.verify.start(deploy_id=deploy_id) default_set_name = 'smoke' default_regex = None @@ -75,7 +71,8 @@ class VerifyCommandsTestCase(test.TestCase): mock_clients().nova().flavors.list.return_value = [ self.flavor1, self.flavor2] tempest_config = tempfile.NamedTemporaryFile() - self.verify.start(deploy_id, tempest_config=tempest_config.name) + self.verify.start(deploy_id=deploy_id, + tempest_config=tempest_config.name) default_set_name = 'smoke' default_regex = None diff --git a/tests/unit/verification/verifiers/test_tempest.py b/tests/unit/verification/verifiers/test_tempest.py index fc3ea83da5..eb4e274187 100644 --- a/tests/unit/verification/verifiers/test_tempest.py +++ b/tests/unit/verification/verifiers/test_tempest.py @@ -13,7 +13,9 @@ # License for the specific language governing permissions and limitations # under the License. +import copy import os +import subprocess import sys import mock @@ -26,172 +28,120 @@ from rally.verification.verifiers.tempest import tempest from tests.unit import test -TEMPEST_PATH = 'rally.verification.verifiers.tempest' +TEMPEST_PATH = "rally.verification.verifiers.tempest" -class TempestTestCase(test.TestCase): - +class BaseTestCase(test.TestCase): def setUp(self): - super(TempestTestCase, self).setUp() + super(BaseTestCase, self).setUp() self.verifier = tempest.Tempest('fake_deploy_id', verification=mock.MagicMock()) - self.verifier.tempest_path = '/tmp' - self.verifier.config_file = '/tmp/tempest.conf' - self.verifier.log_file_raw = '/tmp/subunit.stream' - self.regex = None + self.verifier._path = "/tmp" + self.verifier.config_file = "/tmp/tempest.conf" + self.verifier.log_file_raw = "/tmp/subunit.stream" - @mock.patch('os.path.exists', return_value=True) + +class TempestUtilsTestCase(BaseTestCase): + def test_path(self): + self.assertEqual("/tmp", self.verifier.path()) + self.assertEqual("/tmp/foo", self.verifier.path("foo")) + self.assertEqual("/tmp/foo/bar", self.verifier.path("foo", "bar")) + + @mock.patch("os.path.exists") def test_is_installed(self, mock_exists): - result = self.verifier.is_installed() + # Check that `is_installed` depends on existence of path + # os.path.exists == True => is_installed == True + mock_exists.return_value = True + self.assertTrue(self.verifier.is_installed()) - mock_exists.assert_called_once_with( - os.path.join(self.verifier.tempest_path, '.venv')) - self.assertTrue(result) + # os.path.exists == False => is_installed == False + mock_exists.return_value = False + self.assertFalse(self.verifier.is_installed()) - @mock.patch('rally.verification.verifiers.tempest.tempest.subprocess') - def test__clone(self, mock_sp): - self.verifier._clone() - mock_sp.check_call.assert_called_once_with( - ['git', 'clone', 'https://github.com/openstack/tempest', - tempest.Tempest.tempest_base_path]) + self.assertEqual([mock.call(self.verifier.path(".venv")), + mock.call(self.verifier.path(".venv"))], + mock_exists.call_args_list) - @mock.patch(TEMPEST_PATH + '.tempest.Tempest._initialize_testr') - @mock.patch(TEMPEST_PATH + '.tempest.Tempest._install_venv') - @mock.patch(TEMPEST_PATH + '.tempest.subprocess') - @mock.patch('os.path.exists') - @mock.patch('shutil.copytree') - def test_install( - self, mock_copytree, mock_exists, mock_sp, mock_venv, mock_testr): - mock_exists.side_effect = (False, True, False) - # simulate tempest is clonned but is not installed for current deploy - - self.verifier.install() - mock_copytree.assert_called_once_with( - tempest.Tempest.tempest_base_path, - self.verifier.tempest_path) - mock_sp.check_call.assert_called_once_with( - 'git checkout master; git remote update; git pull', - cwd=os.path.join(self.verifier.tempest_path, 'tempest'), - shell=True) - - @mock.patch('rally.verification.verifiers.tempest.tempest.shutil') - @mock.patch('os.path.exists', return_value=True) - def test_uninstall(self, mock_exists, mock_shutil): - self.verifier.uninstall() - mock_shutil.rmtree.assert_called_once_with(self.verifier.tempest_path) - - @mock.patch(TEMPEST_PATH + '.tempest.os.remove') - @mock.patch(TEMPEST_PATH + '.tempest.Tempest._initialize_testr') - @mock.patch(TEMPEST_PATH + '.tempest.Tempest.run') - @mock.patch(TEMPEST_PATH + '.config.TempestConf') - def test_verify(self, mock_conf, mock_run, mock_testr_init, mock_os): - self.verifier.verify("smoke", None) - mock_conf().generate.assert_called_once_with(self.verifier.config_file) - mock_run.assert_called_once_with("smoke") - - @mock.patch(TEMPEST_PATH + '.tempest.Tempest.env') - @mock.patch(TEMPEST_PATH + '.tempest.subprocess') - @mock.patch(TEMPEST_PATH + '.config.TempestConf') - @mock.patch(TEMPEST_PATH + '.tempest.Tempest.is_configured', - return_value=False) - def test_verify_complex(self, mock_is_configured, mock_conf, - mock_sp, mock_env): - set_name = "compute" - fake_call = ( - "%(venv)s testr run --parallel --subunit tempest.api.%(testr_arg)s" - " | tee %(tempest_path)s/subunit.stream" - " | %(venv)s subunit-2to1" - " | %(venv)s %(tempest_path)s/tools/colorizer.py" % { - "venv": self.verifier.venv_wrapper, - "testr_arg": set_name, - "tempest_path": self.verifier.tempest_path}) - - self.verifier.verify(set_name, None) - mock_conf.assert_called_once_with(self.verifier.deploy_id) - mock_conf().generate.assert_called_once_with(self.verifier.config_file) - self.verifier.verification.start_verifying.assert_called_once_with( - set_name) - - mock_sp.check_call.assert_called_once_with( - fake_call, env=mock_env, cwd=self.verifier.tempest_path, - shell=True) - - @mock.patch('os.environ') - def test__generate_env(self, mock_env): - expected_env = {'PATH': '/some/path'} - mock_env.copy.return_value = expected_env.copy() + @mock.patch("os.environ") + def test_env_missed(self, mock_env): + expected_env = {"PATH": "/some/path"} + mock_env.copy.return_value = copy.deepcopy(expected_env) expected_env.update({ - 'TEMPEST_CONFIG': 'tempest.conf', - 'TEMPEST_CONFIG_DIR': self.verifier.tempest_path, - 'OS_TEST_PATH': os.path.join(self.verifier.tempest_path, - 'tempest/test_discover')}) + "TEMPEST_CONFIG": "tempest.conf", + "TEMPEST_CONFIG_DIR": self.verifier.path(), + "OS_TEST_PATH": self.verifier.path("tempest/test_discover")}) self.assertIsNone(self.verifier._env) - self.verifier._generate_env() + self.assertEqual(expected_env, self.verifier.env) + self.assertTrue(mock_env.copy.called) self.assertEqual(expected_env, self.verifier._env) - @mock.patch('os.path.isdir', return_value=True) - @mock.patch(TEMPEST_PATH + '.tempest.subprocess') + @mock.patch("os.environ") + def test_env_loaded(self, mock_env): + self.verifier._env = {"foo": "bar"} + self.verifier.env + self.assertFalse(mock_env.copy.called) + + @mock.patch("os.path.isdir", return_value=True) + @mock.patch(TEMPEST_PATH + ".tempest.subprocess") @testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version") def test__venv_install_when_venv_exists(self, mock_sp, mock_isdir): self.verifier._install_venv() - mock_isdir.assert_called_once_with( - os.path.join(self.verifier.tempest_path, '.venv')) - self.assertFalse(mock_sp.called) + mock_isdir.assert_called_once_with(self.verifier.path(".venv")) + self.assertFalse(mock_sp.check_output.called) - @mock.patch('os.path.isdir', return_value=False) - @mock.patch(TEMPEST_PATH + '.tempest.subprocess.check_call') + @mock.patch("os.path.isdir", return_value=False) + @mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_output") @testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version") def test__venv_install_when_venv_not_exist(self, mock_sp, mock_isdir): self.verifier._install_venv() - mock_isdir.assert_called_once_with( - os.path.join(self.verifier.tempest_path, '.venv')) + mock_isdir.assert_called_once_with(self.verifier.path(".venv")) mock_sp.assert_has_calls([ - mock.call('python ./tools/install_venv.py', shell=True, - cwd=self.verifier.tempest_path), - mock.call('%s python setup.py install' % + mock.call("python ./tools/install_venv.py", shell=True, + cwd=self.verifier.path()), + mock.call("%s python setup.py install" % self.verifier.venv_wrapper, shell=True, - cwd=self.verifier.tempest_path)]) + cwd=self.verifier.path())]) - @mock.patch('os.path.isdir', return_value=False) + @mock.patch("os.path.isdir", return_value=False) @testtools.skipIf(sys.version_info >= (2, 7), "Incompatible Python Version") def test__venv_install_for_py26_fails(self, mock_isdir): self.assertRaises(exceptions.IncompatiblePythonVersion, self.verifier._install_venv) - mock_isdir.assert_called_once_with( - os.path.join(self.verifier.tempest_path, '.venv')) + mock_isdir.assert_called_once_with(self.verifier.path(".venv")) - @mock.patch('os.path.isdir', return_value=True) - @mock.patch(TEMPEST_PATH + '.tempest.subprocess') + @mock.patch("os.path.isdir", return_value=True) + @mock.patch(TEMPEST_PATH + ".tempest.subprocess") def test__initialize_testr_when_testr_already_initialized( self, mock_sp, mock_isdir): self.verifier._initialize_testr() mock_isdir.assert_called_once_with( - os.path.join(self.verifier.tempest_path, '.testrepository')) + self.verifier.path(".testrepository")) self.assertFalse(mock_sp.called) - @mock.patch('os.path.isdir', return_value=False) - @mock.patch(TEMPEST_PATH + '.tempest.subprocess.check_call') + @mock.patch("os.path.isdir", return_value=False) + @mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_call") def test__initialize_testr_when_testr_not_initialized( self, mock_sp, mock_isdir): self.verifier._initialize_testr() mock_isdir.assert_called_once_with( - os.path.join(self.verifier.tempest_path, '.testrepository')) + self.verifier.path(".testrepository")) mock_sp.assert_called_once_with( '%s testr init' % self.verifier.venv_wrapper, shell=True, - cwd=self.verifier.tempest_path) + cwd=self.verifier.path()) @mock.patch.object(subunit2json, 'main') @mock.patch('os.path.isfile', return_value=False) def test__save_results_without_log_file(self, mock_isfile, mock_parse): self.verifier._save_results() + mock_isfile.assert_called_once_with(self.verifier.log_file_raw) self.assertEqual(0, mock_parse.call_count) @mock.patch('os.path.isfile', return_value=True) @@ -200,12 +150,184 @@ class TempestTestCase(test.TestCase): data = {'total': True, 'test_cases': True} mock_main.return_value = jsonutils.dumps(data) self.verifier.log_file_raw = os.path.join( - os.path.dirname(__file__), - 'subunit.stream') + os.path.dirname(__file__), 'subunit.stream') self.verifier._save_results() mock_isfile.assert_called_once_with(self.verifier.log_file_raw) mock_main.assert_called_once_with( self.verifier.log_file_raw) - self.assertEqual( - 1, self.verifier.verification.finish_verification.call_count) + verification = self.verifier.verification + verification.finish_verification.assert_called_once_with(**data) + + +class TempestInstallAndUninstallTestCase(BaseTestCase): + + @mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_call") + def test__clone_successful(self, mock_sp): + self.verifier._clone() + mock_sp.assert_called_once_with( + ['git', 'clone', 'https://github.com/openstack/tempest', + tempest.Tempest.base_repo]) + + @mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_call") + def test__clone_failed(self, mock_sp): + # Check that `subprocess.CalledProcessError` is not handled by `_clone` + mock_sp.side_effect = subprocess.CalledProcessError(0, None) + + self.assertRaises(subprocess.CalledProcessError, self.verifier._clone) + mock_sp.assert_called_once_with( + ['git', 'clone', 'https://github.com/openstack/tempest', + tempest.Tempest.base_repo]) + + @mock.patch(TEMPEST_PATH + ".tempest.Tempest._initialize_testr") + @mock.patch(TEMPEST_PATH + ".tempest.Tempest._install_venv") + @mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_call") + @mock.patch("shutil.copytree") + @mock.patch(TEMPEST_PATH + ".tempest.Tempest._clone") + @mock.patch("os.path.exists", return_value=False) + def test_install_successful(self, mock_exists, mock_clone, mock_copytree, + mock_sp, mock_install_venv, mock_testr_init): + self.verifier.install() + + self.assertEqual([mock.call(self.verifier.path(".venv")), + mock.call(self.verifier.base_repo), + mock.call(self.verifier.path())], + mock_exists.call_args_list) + mock_clone.assert_called_once_with() + mock_copytree.assert_called_once_with( + self.verifier.base_repo, + self.verifier.path()) + mock_sp.assert_called_once_with( + "git checkout master; git pull", + cwd=self.verifier.path("tempest"), + shell=True) + mock_install_venv.assert_called_once_with() + mock_testr_init.assert_called_once_with() + + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.uninstall") + @mock.patch(TEMPEST_PATH + ".tempest.Tempest._initialize_testr") + @mock.patch(TEMPEST_PATH + ".tempest.Tempest._install_venv") + @mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_call") + @mock.patch("shutil.copytree") + @mock.patch(TEMPEST_PATH + ".tempest.Tempest._clone") + @mock.patch("os.path.exists", return_value=False) + def test_install_failed(self, mock_exists, mock_clone, mock_copytree, + mock_sp, mock_install_venv, mock_testr_init, + mock_uninstall): + mock_sp.side_effect = subprocess.CalledProcessError(0, None) + + self.assertRaises(tempest.TempestSetupFailure, self.verifier.install) + + self.assertEqual([mock.call(self.verifier.path(".venv")), + mock.call(self.verifier.base_repo), + mock.call(self.verifier.path())], + mock_exists.call_args_list) + mock_clone.assert_called_once_with() + mock_copytree.assert_called_once_with( + self.verifier.base_repo, + self.verifier.path()) + mock_sp.assert_called_once_with( + "git checkout master; git pull", + cwd=self.verifier.path("tempest"), + shell=True) + self.assertFalse(mock_install_venv.called) + self.assertFalse(mock_testr_init.called) + mock_uninstall.assert_called_once_with() + + @mock.patch("shutil.rmtree") + @mock.patch("os.path.exists", return_value=True) + def test_uninstall(self, mock_exists, mock_shutil): + self.verifier.uninstall() + mock_exists.assert_called_once_with(self.verifier.path()) + mock_shutil.assert_called_once_with(self.verifier.path()) + + +class TempestVerifyTestCase(BaseTestCase): + def _get_fake_call(self, testr_arg): + return ( + "%(venv)s testr run --parallel --subunit tempest.api.%(testr_arg)s" + " | tee %(tempest_path)s/subunit.stream" + " | %(venv)s subunit-2to1" + " | %(venv)s %(tempest_path)s/tools/colorizer.py" % { + "venv": self.verifier.venv_wrapper, + "testr_arg": testr_arg, + "tempest_path": self.verifier.path()}) + + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.parse_results", + return_value=(None, None)) + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.env") + @mock.patch(TEMPEST_PATH + ".tempest.subprocess") + @mock.patch(TEMPEST_PATH + ".config.TempestConf") + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.is_configured", + return_value=False) + def test_verify_not_configured(self, mock_is_configured, mock_conf, + mock_sp, mock_env, mock_parse_results): + set_name = "compute" + fake_call = self._get_fake_call(set_name) + + self.verifier.verify(set_name, None) + + self.assertEqual(2, mock_is_configured.call_count) + mock_conf.assert_called_once_with(self.verifier.deploy_id) + mock_conf().generate.assert_called_once_with(self.verifier.config_file) + self.verifier.verification.start_verifying.assert_called_once_with( + set_name) + + mock_sp.check_call.assert_called_once_with( + fake_call, env=mock_env, cwd=self.verifier.path(), + shell=True) + mock_parse_results.assert_called_once_with() + + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.parse_results", + return_value=(None, None)) + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.env") + @mock.patch(TEMPEST_PATH + ".tempest.subprocess") + @mock.patch(TEMPEST_PATH + ".config.TempestConf") + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.is_configured", + return_value=True) + def test_verify_when_tempest_configured(self, mock_is_configured, + mock_conf, mock_sp, mock_env, + mock_parse_results): + set_name = "identity" + fake_call = self._get_fake_call(set_name) + + self.verifier.verify(set_name, None) + + mock_is_configured.assert_called_once_with() + self.assertFalse(mock_conf.called) + self.assertFalse(mock_conf().generate.called) + self.verifier.verification.start_verifying.assert_called_once_with( + set_name) + + mock_sp.check_call.assert_called_once_with( + fake_call, env=mock_env, cwd=self.verifier.path(), + shell=True) + mock_parse_results.assert_called_once_with() + + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.parse_results", + return_value=(None, None)) + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.env") + @mock.patch(TEMPEST_PATH + ".tempest.subprocess") + @mock.patch(TEMPEST_PATH + ".config.TempestConf") + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.is_configured", + return_value=True) + def test_verify_failed_and_tempest_is_configured( + self, mock_is_configured, mock_conf, mock_sp, mock_env, + mock_parse_results): + set_name = "identity" + fake_call = self._get_fake_call(set_name) + mock_sp.side_effect = subprocess.CalledProcessError + + self.verifier.verify(set_name, None) + + mock_is_configured.assert_called_once_with() + self.assertFalse(mock_conf.called) + self.assertFalse(mock_conf().generate.called) + self.verifier.verification.start_verifying.assert_called_once_with( + set_name) + + mock_sp.check_call.assert_called_once_with( + fake_call, env=mock_env, cwd=self.verifier.path(), + shell=True) + self.assertTrue(mock_parse_results.called) + self.verifier.verification.set_failed.assert_called_once_with() diff --git a/tools/rally.bash_completion b/tools/rally.bash_completion index a3caa227df..665770b38d 100644 --- a/tools/rally.bash_completion +++ b/tools/rally.bash_completion @@ -9,6 +9,7 @@ _rally() OPTS["info_find"]="--query" OPTS["use_deployment"]="--uuid --name" OPTS["use_task"]="--uuid" + OPTS["use_verification"]="--uuid" OPTS["task_abort"]="--uuid" OPTS["task_delete"]="--force --uuid" OPTS["task_detailed"]="--uuid --iterations-data" @@ -29,7 +30,7 @@ _rally() OPTS["verify_list"]="" OPTS["verify_results"]="--uuid --html --json --pprint --output-file" OPTS["verify_show"]="--uuid --sort-by --detailed" - OPTS["verify_start"]="--deploy-id --set --regex --tempest-config" + OPTS["verify_start"]="--deploy-id --set --regex --tempest-config --no-use" OPTS["deployment_check"]="--uuid" OPTS["deployment_config"]="--uuid --json --pprint" OPTS["deployment_create"]="--name --fromenv --filename --no-use"