diff --git a/etc/rally.bash_completion b/etc/rally.bash_completion index 636bca60fd..47baa48565 100644 --- a/etc/rally.bash_completion +++ b/etc/rally.bash_completion @@ -56,7 +56,7 @@ _rally() OPTS["verify_results"]="--uuid --html --json --output-file" OPTS["verify_show"]="--uuid --sort-by --detailed" OPTS["verify_showconfig"]="--deployment" - OPTS["verify_start"]="--deployment --set --regex --tempest-config --no-use --system-wide-install" + OPTS["verify_start"]="--deployment --set --regex --tests-file --tempest-config --no-use --system-wide-install" OPTS["verify_uninstall"]="--deployment" OPTS["verify_use"]="--verification" @@ -87,4 +87,4 @@ _rally() return 0 } -complete -o filenames -F _rally rally \ No newline at end of file +complete -o filenames -F _rally rally diff --git a/rally/api.py b/rally/api.py index 335087d3a0..52b4614cf9 100644 --- a/rally/api.py +++ b/rally/api.py @@ -288,13 +288,14 @@ class Task(object): class Verification(object): @classmethod - def verify(cls, deployment, set_name, regex, tempest_config, - system_wide_install=False): + def verify(cls, deployment, set_name, regex, tests_file, + tempest_config, system_wide_install=False): """Start verifying. :param deployment: UUID or name of a deployment. :param set_name: Valid name of tempest test set. :param regex: Regular expression of test + :param tests_file: Path to a file with a list of Tempest tests :param tempest_config: User specified Tempest config file :param system_wide_install: Use virtualenv else run tests in local environment @@ -309,7 +310,8 @@ class Verification(object): LOG.info("Starting verification of deployment: %s" % deployment_uuid) verification.set_running() - verifier.verify(set_name=set_name, regex=regex) + verifier.verify(set_name=set_name, regex=regex, + tests_file=tests_file) return verification diff --git a/rally/cli/commands/verify.py b/rally/cli/commands/verify.py index 5275365b36..7b7d32549d 100644 --- a/rally/cli/commands/verify.py +++ b/rally/cli/commands/verify.py @@ -49,6 +49,9 @@ class VerifyCommands(object): list(consts.TempestTestsAPI))) @cliutils.args("--regex", dest="regex", type=str, required=False, help="Regular expression of test.") + @cliutils.args("--tests-file", dest="tests_file", type=str, + help="Path to a file with a list of Tempest tests", + required=False) @cliutils.args("--tempest-config", dest="tempest_config", type=str, required=False, help="User specified Tempest config file location") @@ -59,13 +62,14 @@ class VerifyCommands(object): required=False, action="store_true") @envutils.with_default_deployment(cli_arg_name="deployment") def start(self, set_name="", deployment=None, regex=None, - tempest_config=None, do_use=True, + tests_file=None, tempest_config=None, do_use=True, system_wide_install=False): """Start set of tests. :param set_name: Name of tempest test set :param deployment: UUID or name of a deployment :param regex: Regular expression of test + :param tests_file: Path to a file with a list of Tempest tests :param tempest_config: User specified Tempest config file location :param do_use: Use new task as default for future operations :param system_wide_install: Use virtualenv else run tests in @@ -73,18 +77,33 @@ class VerifyCommands(object): """ if regex and set_name: - raise exceptions.InvalidArgumentsException("set_name and regex " - "are not compatible") - if not (regex or set_name): + raise exceptions.InvalidArgumentsException( + "Arguments set_name and regex are not compatible") + + if tests_file and set_name: + raise exceptions.InvalidArgumentsException( + "Arguments tests_file and set_name are not compatible") + + if tests_file and regex: + raise exceptions.InvalidArgumentsException( + "Arguments tests_file and regex are not compatible") + + if not (regex or set_name or tests_file): set_name = "full" + if set_name and set_name not in (list(consts.TempestTestsSets) + list(consts.TempestTestsAPI)): print("Sorry, but there are no desired Tempest test set. Please, " "choose from: %s" % ", ".join(list(consts.TempestTestsSets) + list(consts.TempestTestsAPI))) return (1) + + if tests_file and not os.path.exists(tests_file): + print("File '%s' not found" % tests_file) + return (1) + verification = api.Verification.verify(deployment, set_name, regex, - tempest_config, + tests_file, tempest_config, system_wide_install) if do_use: self.use(verification["uuid"]) diff --git a/rally/verification/tempest/tempest.py b/rally/verification/tempest/tempest.py index c156bc462f..5e43de5c15 100644 --- a/rally/verification/tempest/tempest.py +++ b/rally/verification/tempest/tempest.py @@ -294,7 +294,7 @@ class Tempest(object): shutil.rmtree(self.path()) @utils.log_verification_wrapper(LOG.info, _("Run verification.")) - def _prepare_and_run(self, set_name, regex): + def _prepare_and_run(self, set_name, regex, tests_file): if not self.is_configured(): self.generate_config_file() @@ -303,6 +303,8 @@ class Tempest(object): else: if set_name in consts.TempestTestsAPI: testr_arg = "tempest.api.%s" % set_name + elif tests_file: + testr_arg = "--load-list %s" % os.path.abspath(tests_file) else: testr_arg = set_name or regex @@ -393,8 +395,8 @@ class Tempest(object): else: self.verification.set_failed() - def verify(self, set_name, regex): - self._prepare_and_run(set_name, regex) + def verify(self, set_name, regex, tests_file): + self._prepare_and_run(set_name, regex, tests_file) self._save_results() def import_results(self, set_name, log_file): diff --git a/tests/unit/cli/commands/test_verify.py b/tests/unit/cli/commands/test_verify.py index df6fc430df..72a5009ec5 100644 --- a/tests/unit/cli/commands/test_verify.py +++ b/tests/unit/cli/commands/test_verify.py @@ -58,9 +58,11 @@ class VerifyCommandsTestCase(test.TestCase): self.verify.start(deployment=deployment_id, do_use=False) default_set_name = "full" default_regex = None + default_tests_file = None mock_verification_verify.assert_called_once_with( - deployment_id, default_set_name, default_regex, None, False) + deployment_id, default_set_name, default_regex, + default_tests_file, None, False) @mock.patch("rally.osclients.Clients") @mock.patch("rally.api.Verification.verify") @@ -76,12 +78,25 @@ class VerifyCommandsTestCase(test.TestCase): tempest_config=tempest_config.name, do_use=False) default_set_name = "full" default_regex = None + default_tests_file = None mock_verification_verify.assert_called_once_with( deployment_id, default_set_name, default_regex, - tempest_config.name, False) + default_tests_file, tempest_config.name, False) tempest_config.close() + @mock.patch("rally.api.Verification.verify") + @mock.patch("os.path.exists", return_value=True) + def test_start_with_tests_file_specified(self, mock_exists, + mock_verification_verify): + deployment_id = "f05645f9-b3d1-4be4-ae63-ae6ea6d89f17" + tests_file = "/path/to/tests/file" + self.verify.start(deployment=deployment_id, + tests_file=tests_file, do_use=False) + + mock_verification_verify.assert_called_once_with( + deployment_id, "", None, tests_file, None, False) + @mock.patch("rally.api.Verification.verify") def test_start_with_wrong_set_name(self, mock_verification_verify): deployment_id = "f2009aae-6ef3-468e-96b2-3c987d584010" diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index c2b925345c..64cc071991 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -348,11 +348,12 @@ class VerificationAPITestCase(BaseDeploymentTestCase): mock_tempest.return_value = self.tempest self.tempest.is_installed.return_value = True - api.Verification.verify(self.deployment_uuid, "smoke", None, None) + api.Verification.verify( + self.deployment_uuid, "smoke", None, None, None) self.tempest.is_installed.assert_called_once_with() - self.tempest.verify.assert_called_once_with(set_name="smoke", - regex=None) + self.tempest.verify.assert_called_once_with( + set_name="smoke", regex=None, tests_file=None) @mock.patch("rally.api.objects.Deployment.get") @mock.patch("rally.api.objects.Verification") @@ -363,12 +364,29 @@ class VerificationAPITestCase(BaseDeploymentTestCase): mock_deployment_get.return_value = {"uuid": self.deployment_uuid} mock_tempest.return_value = self.tempest self.tempest.is_installed.return_value = False - api.Verification.verify(self.deployment_uuid, "smoke", None, None) + api.Verification.verify( + self.deployment_uuid, "smoke", None, None, None) self.tempest.is_installed.assert_called_once_with() self.tempest.install.assert_called_once_with() - self.tempest.verify.assert_called_once_with(set_name="smoke", - regex=None) + self.tempest.verify.assert_called_once_with( + set_name="smoke", regex=None, tests_file=None) + + @mock.patch("os.path.exists", return_value=True) + @mock.patch("rally.api.objects.Deployment.get") + @mock.patch("rally.api.objects.Verification") + @mock.patch("rally.verification.tempest.tempest.Tempest") + def test_verify_tests_file_specified(self, mock_tempest, mock_verification, + mock_deployment_get, mock_exists): + mock_deployment_get.return_value = {"uuid": self.deployment_uuid} + mock_tempest.return_value = self.tempest + self.tempest.is_installed.return_value = True + tests_file = "/path/to/tests/file" + api.Verification.verify( + self.deployment_uuid, "", None, tests_file, None) + + self.tempest.verify.assert_called_once_with( + set_name="", regex=None, tests_file=tests_file) @mock.patch("rally.common.objects.Deployment.get") @mock.patch("rally.api.objects.Verification") diff --git a/tests/unit/verification/test_tempest.py b/tests/unit/verification/test_tempest.py index d524b4f7da..b7683a23ff 100644 --- a/tests/unit/verification/test_tempest.py +++ b/tests/unit/verification/test_tempest.py @@ -323,14 +323,15 @@ class TempestInstallAndUninstallTestCase(BaseTestCase): class TempestVerifyTestCase(BaseTestCase): - def _get_fake_call(self, testr_arg): + def _get_fake_call(self, testr_arg, is_set=True): return ( - "%(venv)s testr run --parallel --subunit tempest.api.%(testr_arg)s" + "%(venv)s testr run --parallel --subunit %(testr_arg)s" " | tee %(tempest_path)s/subunit.stream" " | %(venv)s subunit-2to1" " | %(venv)s %(tempest_path)s/tools/colorizer.py" % { "venv": self.verifier.venv_wrapper, - "testr_arg": testr_arg, + "testr_arg": ("tempest.api." if is_set + else "--load-list ") + testr_arg, "tempest_path": self.verifier.path()}) @mock.patch(TEMPEST_PATH + ".tempest.Tempest.parse_results", @@ -349,7 +350,7 @@ class TempestVerifyTestCase(BaseTestCase): set_name = "compute" fake_call = self._get_fake_call(set_name) - self.verifier.verify(set_name, None) + self.verifier.verify(set_name, None, None) self.assertEqual(2, mock_tempest_is_configured.call_count) mock_tempest_config.assert_called_once_with(self.verifier.deployment) @@ -379,7 +380,7 @@ class TempestVerifyTestCase(BaseTestCase): set_name = "identity" fake_call = self._get_fake_call(set_name) - self.verifier.verify(set_name, None) + self.verifier.verify(set_name, None, None) mock_tempest_is_configured.assert_called_once_with() self.assertFalse(mock_tempest_config.called) @@ -408,7 +409,7 @@ class TempestVerifyTestCase(BaseTestCase): fake_call = self._get_fake_call(set_name) mock_subprocess.side_effect = subprocess.CalledProcessError - self.verifier.verify(set_name, None) + self.verifier.verify(set_name, None, None) mock_tempest_is_configured.assert_called_once_with() self.assertFalse(mock_tempest_config.called) @@ -422,6 +423,27 @@ class TempestVerifyTestCase(BaseTestCase): self.assertTrue(mock_tempest_parse_results.called) self.verifier.verification.set_failed.assert_called_once_with() + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.parse_results", + return_value=(None, None)) + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.env") + @mock.patch(TEMPEST_PATH + ".tempest.subprocess") + @mock.patch(TEMPEST_PATH + ".config.TempestResourcesContext") + @mock.patch(TEMPEST_PATH + ".tempest.Tempest.is_configured", + return_value=True) + def test_verify_tests_file_specified( + self, mock_tempest_is_configured, mock_tempest_resources_context, + mock_subprocess, mock_tempest_env, mock_tempest_parse_results): + tests_file = "/path/to/tests/file" + fake_call = self._get_fake_call(tests_file, is_set=False) + + self.verifier.verify("", None, tests_file) + self.verifier.verification.start_verifying.assert_called_once_with("") + + mock_subprocess.check_call.assert_called_once_with( + fake_call, env=mock_tempest_env, cwd=self.verifier.path(), + shell=True) + mock_tempest_parse_results.assert_called_once_with(None) + def test_import_results(self): set_name = "identity" log_file = "log_file"