Merge "Add parameter --tests-file to 'verify start'"

etc/rally.bash_completion

@@ -56,7 +56,7 @@ _rally()
     OPTS["verify_results"]="--uuid --html --json --output-file"
     OPTS["verify_show"]="--uuid --sort-by --detailed"
     OPTS["verify_showconfig"]="--deployment"
-    OPTS["verify_start"]="--deployment --set --regex --tempest-config --no-use --system-wide-install"
+    OPTS["verify_start"]="--deployment --set --regex --tests-file --tempest-config --no-use --system-wide-install"
     OPTS["verify_uninstall"]="--deployment"
     OPTS["verify_use"]="--verification"
 

rally/api.py

@@ -288,13 +288,14 @@ class Task(object):
 class Verification(object):
 
     @classmethod
-    def verify(cls, deployment, set_name, regex, tempest_config,
+    def verify(cls, deployment, set_name, regex, tests_file,
-               system_wide_install=False):
+               tempest_config, system_wide_install=False):
         """Start verifying.
 
         :param deployment: UUID or name of a deployment.
         :param set_name: Valid name of tempest test set.
         :param regex: Regular expression of test
+        :param tests_file: Path to a file with a list of Tempest tests
         :param tempest_config: User specified Tempest config file
         :param system_wide_install: Use virtualenv else run tests in local
                                     environment
@@ -309,7 +310,8 @@ class Verification(object):
         LOG.info("Starting verification of deployment: %s" % deployment_uuid)
 
         verification.set_running()
-        verifier.verify(set_name=set_name, regex=regex)
+        verifier.verify(set_name=set_name, regex=regex,
+                        tests_file=tests_file)
 
         return verification
 

rally/cli/commands/verify.py

@@ -49,6 +49,9 @@ class VerifyCommands(object):
                         list(consts.TempestTestsAPI)))
     @cliutils.args("--regex", dest="regex", type=str, required=False,
                    help="Regular expression of test.")
+    @cliutils.args("--tests-file", dest="tests_file", type=str,
+                   help="Path to a file with a list of Tempest tests",
+                   required=False)
     @cliutils.args("--tempest-config", dest="tempest_config", type=str,
                    required=False,
                    help="User specified Tempest config file location")
@@ -59,13 +62,14 @@ class VerifyCommands(object):
                    required=False, action="store_true")
     @envutils.with_default_deployment(cli_arg_name="deployment")
     def start(self, set_name="", deployment=None, regex=None,
-              tempest_config=None, do_use=True,
+              tests_file=None, tempest_config=None, do_use=True,
               system_wide_install=False):
         """Start set of tests.
 
         :param set_name: Name of tempest test set
         :param deployment: UUID or name of a deployment
         :param regex: Regular expression of test
+        :param tests_file: Path to a file with a list of Tempest tests
         :param tempest_config: User specified Tempest config file location
         :param do_use: Use new task as default for future operations
         :param system_wide_install: Use virtualenv else run tests in
@@ -73,18 +77,33 @@ class VerifyCommands(object):
         """
 
         if regex and set_name:
-            raise exceptions.InvalidArgumentsException("set_name and regex "
+            raise exceptions.InvalidArgumentsException(
-                                                       "are not compatible")
+                "Arguments set_name and regex are not compatible")
-        if not (regex or set_name):
+
+        if tests_file and set_name:
+            raise exceptions.InvalidArgumentsException(
+                "Arguments tests_file and set_name are not compatible")
+
+        if tests_file and regex:
+            raise exceptions.InvalidArgumentsException(
+                "Arguments tests_file and regex are not compatible")
+
+        if not (regex or set_name or tests_file):
             set_name = "full"
+
         if set_name and set_name not in (list(consts.TempestTestsSets) +
                                          list(consts.TempestTestsAPI)):
             print("Sorry, but there are no desired Tempest test set. Please, "
                   "choose from: %s" % ", ".join(list(consts.TempestTestsSets) +
                                                 list(consts.TempestTestsAPI)))
             return (1)
+
+        if tests_file and not os.path.exists(tests_file):
+            print("File '%s' not found" % tests_file)
+            return (1)
+
         verification = api.Verification.verify(deployment, set_name, regex,
-                                               tempest_config,
+                                               tests_file, tempest_config,
                                                system_wide_install)
         if do_use:
             self.use(verification["uuid"])

rally/verification/tempest/tempest.py

@@ -294,7 +294,7 @@ class Tempest(object):
             shutil.rmtree(self.path())
 
     @utils.log_verification_wrapper(LOG.info, _("Run verification."))
-    def _prepare_and_run(self, set_name, regex):
+    def _prepare_and_run(self, set_name, regex, tests_file):
         if not self.is_configured():
             self.generate_config_file()
 
@@ -303,6 +303,8 @@ class Tempest(object):
         else:
             if set_name in consts.TempestTestsAPI:
                 testr_arg = "tempest.api.%s" % set_name
+            elif tests_file:
+                testr_arg = "--load-list %s" % os.path.abspath(tests_file)
             else:
                 testr_arg = set_name or regex
 
@@ -393,8 +395,8 @@ class Tempest(object):
         else:
             self.verification.set_failed()
 
-    def verify(self, set_name, regex):
+    def verify(self, set_name, regex, tests_file):
-        self._prepare_and_run(set_name, regex)
+        self._prepare_and_run(set_name, regex, tests_file)
         self._save_results()
 
     def import_results(self, set_name, log_file):

tests/unit/cli/commands/test_verify.py

@@ -58,9 +58,11 @@ class VerifyCommandsTestCase(test.TestCase):
         self.verify.start(deployment=deployment_id, do_use=False)
         default_set_name = "full"
         default_regex = None
+        default_tests_file = None
 
         mock_verification_verify.assert_called_once_with(
-            deployment_id, default_set_name, default_regex, None, False)
+            deployment_id, default_set_name, default_regex,
+            default_tests_file, None, False)
 
     @mock.patch("rally.osclients.Clients")
     @mock.patch("rally.api.Verification.verify")
@@ -76,12 +78,25 @@ class VerifyCommandsTestCase(test.TestCase):
                           tempest_config=tempest_config.name, do_use=False)
         default_set_name = "full"
         default_regex = None
+        default_tests_file = None
 
         mock_verification_verify.assert_called_once_with(
             deployment_id, default_set_name, default_regex,
-            tempest_config.name, False)
+            default_tests_file, tempest_config.name, False)
         tempest_config.close()
 
+    @mock.patch("rally.api.Verification.verify")
+    @mock.patch("os.path.exists", return_value=True)
+    def test_start_with_tests_file_specified(self, mock_exists,
+                                             mock_verification_verify):
+        deployment_id = "f05645f9-b3d1-4be4-ae63-ae6ea6d89f17"
+        tests_file = "/path/to/tests/file"
+        self.verify.start(deployment=deployment_id,
+                          tests_file=tests_file, do_use=False)
+
+        mock_verification_verify.assert_called_once_with(
+            deployment_id, "", None, tests_file, None, False)
+
     @mock.patch("rally.api.Verification.verify")
     def test_start_with_wrong_set_name(self, mock_verification_verify):
         deployment_id = "f2009aae-6ef3-468e-96b2-3c987d584010"

tests/unit/test_api.py

@@ -348,11 +348,12 @@ class VerificationAPITestCase(BaseDeploymentTestCase):
 
         mock_tempest.return_value = self.tempest
         self.tempest.is_installed.return_value = True
-        api.Verification.verify(self.deployment_uuid, "smoke", None, None)
+        api.Verification.verify(
+            self.deployment_uuid, "smoke", None, None, None)
 
         self.tempest.is_installed.assert_called_once_with()
-        self.tempest.verify.assert_called_once_with(set_name="smoke",
+        self.tempest.verify.assert_called_once_with(
-                                                    regex=None)
+            set_name="smoke", regex=None, tests_file=None)
 
     @mock.patch("rally.api.objects.Deployment.get")
     @mock.patch("rally.api.objects.Verification")
@@ -363,12 +364,29 @@ class VerificationAPITestCase(BaseDeploymentTestCase):
         mock_deployment_get.return_value = {"uuid": self.deployment_uuid}
         mock_tempest.return_value = self.tempest
         self.tempest.is_installed.return_value = False
-        api.Verification.verify(self.deployment_uuid, "smoke", None, None)
+        api.Verification.verify(
+            self.deployment_uuid, "smoke", None, None, None)
 
         self.tempest.is_installed.assert_called_once_with()
         self.tempest.install.assert_called_once_with()
-        self.tempest.verify.assert_called_once_with(set_name="smoke",
+        self.tempest.verify.assert_called_once_with(
-                                                    regex=None)
+            set_name="smoke", regex=None, tests_file=None)
+
+    @mock.patch("os.path.exists", return_value=True)
+    @mock.patch("rally.api.objects.Deployment.get")
+    @mock.patch("rally.api.objects.Verification")
+    @mock.patch("rally.verification.tempest.tempest.Tempest")
+    def test_verify_tests_file_specified(self, mock_tempest, mock_verification,
+                                         mock_deployment_get, mock_exists):
+        mock_deployment_get.return_value = {"uuid": self.deployment_uuid}
+        mock_tempest.return_value = self.tempest
+        self.tempest.is_installed.return_value = True
+        tests_file = "/path/to/tests/file"
+        api.Verification.verify(
+            self.deployment_uuid, "", None, tests_file, None)
+
+        self.tempest.verify.assert_called_once_with(
+            set_name="", regex=None, tests_file=tests_file)
 
     @mock.patch("rally.common.objects.Deployment.get")
     @mock.patch("rally.api.objects.Verification")

tests/unit/verification/test_tempest.py

@@ -323,14 +323,15 @@ class TempestInstallAndUninstallTestCase(BaseTestCase):
 
 
 class TempestVerifyTestCase(BaseTestCase):
-    def _get_fake_call(self, testr_arg):
+    def _get_fake_call(self, testr_arg, is_set=True):
         return (
-            "%(venv)s testr run --parallel --subunit tempest.api.%(testr_arg)s"
+            "%(venv)s testr run --parallel --subunit %(testr_arg)s"
             " | tee %(tempest_path)s/subunit.stream"
             " | %(venv)s subunit-2to1"
             " | %(venv)s %(tempest_path)s/tools/colorizer.py" % {
                 "venv": self.verifier.venv_wrapper,
-                "testr_arg": testr_arg,
+                "testr_arg": ("tempest.api." if is_set
+                              else "--load-list ") + testr_arg,
                 "tempest_path": self.verifier.path()})
 
     @mock.patch(TEMPEST_PATH + ".tempest.Tempest.parse_results",
@@ -349,7 +350,7 @@ class TempestVerifyTestCase(BaseTestCase):
         set_name = "compute"
         fake_call = self._get_fake_call(set_name)
 
-        self.verifier.verify(set_name, None)
+        self.verifier.verify(set_name, None, None)
 
         self.assertEqual(2, mock_tempest_is_configured.call_count)
         mock_tempest_config.assert_called_once_with(self.verifier.deployment)
@@ -379,7 +380,7 @@ class TempestVerifyTestCase(BaseTestCase):
         set_name = "identity"
         fake_call = self._get_fake_call(set_name)
 
-        self.verifier.verify(set_name, None)
+        self.verifier.verify(set_name, None, None)
 
         mock_tempest_is_configured.assert_called_once_with()
         self.assertFalse(mock_tempest_config.called)
@@ -408,7 +409,7 @@ class TempestVerifyTestCase(BaseTestCase):
         fake_call = self._get_fake_call(set_name)
         mock_subprocess.side_effect = subprocess.CalledProcessError
 
-        self.verifier.verify(set_name, None)
+        self.verifier.verify(set_name, None, None)
 
         mock_tempest_is_configured.assert_called_once_with()
         self.assertFalse(mock_tempest_config.called)
@@ -422,6 +423,27 @@ class TempestVerifyTestCase(BaseTestCase):
         self.assertTrue(mock_tempest_parse_results.called)
         self.verifier.verification.set_failed.assert_called_once_with()
 
+    @mock.patch(TEMPEST_PATH + ".tempest.Tempest.parse_results",
+                return_value=(None, None))
+    @mock.patch(TEMPEST_PATH + ".tempest.Tempest.env")
+    @mock.patch(TEMPEST_PATH + ".tempest.subprocess")
+    @mock.patch(TEMPEST_PATH + ".config.TempestResourcesContext")
+    @mock.patch(TEMPEST_PATH + ".tempest.Tempest.is_configured",
+                return_value=True)
+    def test_verify_tests_file_specified(
+            self, mock_tempest_is_configured, mock_tempest_resources_context,
+            mock_subprocess, mock_tempest_env, mock_tempest_parse_results):
+        tests_file = "/path/to/tests/file"
+        fake_call = self._get_fake_call(tests_file, is_set=False)
+
+        self.verifier.verify("", None, tests_file)
+        self.verifier.verification.start_verifying.assert_called_once_with("")
+
+        mock_subprocess.check_call.assert_called_once_with(
+            fake_call, env=mock_tempest_env, cwd=self.verifier.path(),
+            shell=True)
+        mock_tempest_parse_results.assert_called_once_with(None)
+
     def test_import_results(self):
         set_name = "identity"
         log_file = "log_file"