[Verify] Adding possibility to specify concurrency for tests

The `rally verify start` command doesn't allow us to specify
concurrency for Tempest tests. The tests always are run in parallel
mode and use concurrency equal to the count of CPU cores. However,
users may want to use specific concurrency for running tests based
on their deployments and available resources.

This patch adds possibility to specify desired concurrency for tests.

Closes-Bug: #1524968

Change-Id: I3ba2c987c739b13bb03005b1988ee6bd01e8a08d

etc/rally.bash_completion

@@ -56,7 +56,7 @@ _rally()
     OPTS["verify_results"]="--uuid --html --json --output-file"
     OPTS["verify_show"]="--uuid --sort-by --detailed"
     OPTS["verify_showconfig"]="--deployment"
-    OPTS["verify_start"]="--deployment --set --regex --tests-file --tempest-config --no-use --system-wide-install"
+    OPTS["verify_start"]="--deployment --set --regex --tests-file --tempest-config --no-use --system-wide-install --concurrency"
     OPTS["verify_uninstall"]="--deployment"
     OPTS["verify_use"]="--verification"
 

rally/api.py

@@ -358,7 +358,7 @@ class Verification(object):
 
     @classmethod
     def verify(cls, deployment, set_name, regex, tests_file,
-               tempest_config, system_wide_install=False):
+               tempest_config, system_wide_install=False, concur=0):
         """Start verifying.
 
         :param deployment: UUID or name of a deployment
@@ -370,9 +370,10 @@ class Verification(object):
                                     when installing Tempest; whether or not to
                                     use the local env instead of the Tempest
                                     virtual env when running the tests
+        :param concur: How many processes to use to run Tempest tests.
+                       The default value (0) auto-detects CPU count
         :returns: Verification object
         """
-
         deployment_uuid = objects.Deployment.get(deployment)["uuid"]
         verification = objects.Verification(deployment_uuid=deployment_uuid)
         verifier = tempest.Tempest(deployment_uuid, verification=verification,
@@ -389,7 +390,7 @@ class Verification(object):
         LOG.info("Starting verification of deployment: %s" % deployment_uuid)
         verification.set_running()
         verifier.verify(set_name=set_name, regex=regex,
-                        tests_file=tests_file)
+                        tests_file=tests_file, concur=concur)
 
         return verification
 

rally/cli/commands/verify.py

@@ -63,10 +63,13 @@ class VerifyCommands(object):
                         "requirements have to be already installed in "
                         "the local env!",
                    required=False, action="store_true")
+    @cliutils.args("--concurrency", dest="concur", type=int, required=False,
+                   help="How many processes to use to run Tempest tests. "
+                        "The default value (0) auto-detects your CPU count")
     @envutils.with_default_deployment(cli_arg_name="deployment")
     def start(self, set_name="", deployment=None, regex=None,
               tests_file=None, tempest_config=None, do_use=True,
-              system_wide_install=False):
+              system_wide_install=False, concur=0):
         """Start set of tests.
 
         :param set_name: Name of tempest test set
@@ -79,6 +82,8 @@ class VerifyCommands(object):
                                     when installing Tempest; whether or not to
                                     use the local env instead of the Tempest
                                     virtual env when running the tests
+        :param concur: How many processes to use to run Tempest tests.
+                       The default value (0) auto-detects CPU count
         """
 
         if regex and set_name:
@@ -109,7 +114,7 @@ class VerifyCommands(object):
 
         verification = api.Verification.verify(deployment, set_name, regex,
                                                tests_file, tempest_config,
-                                               system_wide_install)
+                                               system_wide_install, concur)
         if do_use:
             self.use(verification["uuid"])
 

rally/verification/tempest/tempest.py

@@ -297,54 +297,48 @@ class Tempest(object):
             shutil.rmtree(self.path())
 
     @logging.log_verification_wrapper(LOG.info, _("Run verification."))
-    def _prepare_and_run(self, set_name, regex, tests_file):
+    def _prepare_and_run(self, set_name, regex, tests_file, concur):
         if not self.is_configured():
             self.generate_config_file()
 
+        testr_args = "--concurrency %d" % concur
+
         if set_name == "full":
-            testr_arg = ""
+            pass
+        elif set_name in consts.TempestTestsAPI:
+            testr_args += " tempest.api.%s" % set_name
+        elif regex:
+            testr_args += " %s" % regex
         else:
-            if set_name in consts.TempestTestsAPI:
+            testr_args += " --load-list %s" % os.path.abspath(tests_file)
-                testr_arg = "tempest.api.%s" % set_name
-            elif tests_file:
-                testr_arg = "--load-list %s" % os.path.abspath(tests_file)
-            else:
-                testr_arg = set_name or regex
 
         self.verification.start_verifying(set_name)
         try:
-            self.run(testr_arg)
+            self.run(testr_args)
         except subprocess.CalledProcessError:
-            LOG.info(_("Test set '%s' has been finished with errors. "
+            LOG.info(_("Test run has been finished with errors. "
-                       "Check logs for details.") % set_name)
+                       "Check logs for details."))
 
-    def run(self, testr_arg=None, log_file=None, tempest_conf=None):
+    def run(self, testr_args="", log_file=None, tempest_conf=None):
-        """Launch tempest with given arguments
+        """Run Tempest.
 
-        :param testr_arg: argument which will be transmitted into testr
+        :param testr_args: Arguments which will be passed to testr
-        :type testr_arg: str
+        :param log_file: Path to a file for raw subunit stream logs.
-        :param log_file: file name for raw subunit results of tests. If not
+                         If not specified, the value from "self.log_file_raw"
-                         specified, value from "self.log_file_raw"
+                         will be used as the path to the file
-                         will be chosen.
+        :param tempest_conf: User specified Tempest config file location
-        :type log_file: str
-        :param tempest_conf: User specified tempest.conf location
-        :type tempest_conf: str
-
-        :raises subprocess.CalledProcessError: if tests has been finished
-                                               with error.
         """
-
         if tempest_conf and os.path.isfile(tempest_conf):
             self.config_file = tempest_conf
         LOG.info(_("Tempest config file: %s") % self.config_file)
 
         test_cmd = (
-            "%(venv)s testr run --parallel --subunit %(arg)s "
+            "%(venv)s testr run --subunit --parallel %(testr_args)s "
             "| tee %(log_file)s "
             "| %(venv)s subunit-trace -f -n" %
             {
                 "venv": self.venv_wrapper,
-                "arg": testr_arg,
+                "testr_args": testr_args,
                 "log_file": log_file or self.log_file_raw
             })
         LOG.debug("Test(s) started by the command: %s" % test_cmd)
@@ -393,8 +387,8 @@ class Tempest(object):
         else:
             self.verification.set_failed()
 
-    def verify(self, set_name, regex, tests_file):
+    def verify(self, set_name, regex, tests_file, concur):
-        self._prepare_and_run(set_name, regex, tests_file)
+        self._prepare_and_run(set_name, regex, tests_file, concur)
         self._save_results()
 
     def import_results(self, set_name, log_file):

tests/unit/cli/commands/test_verify.py

@@ -60,7 +60,7 @@ class VerifyCommandsTestCase(test.TestCase):
 
         mock_verification_verify.assert_called_once_with(
             deployment_id, default_set_name, default_regex,
-            default_tests_file, None, False)
+            default_tests_file, None, False, 0)
 
     @mock.patch("rally.osclients.Clients")
     @mock.patch("rally.api.Verification.verify")
@@ -80,7 +80,7 @@ class VerifyCommandsTestCase(test.TestCase):
 
         mock_verification_verify.assert_called_once_with(
             deployment_id, default_set_name, default_regex,
-            default_tests_file, tempest_config.name, False)
+            default_tests_file, tempest_config.name, False, 0)
         tempest_config.close()
 
     @mock.patch("rally.api.Verification.verify")
@@ -93,7 +93,7 @@ class VerifyCommandsTestCase(test.TestCase):
                           tests_file=tests_file, do_use=False)
 
         mock_verification_verify.assert_called_once_with(
-            deployment_id, "", None, tests_file, None, False)
+            deployment_id, "", None, tests_file, None, False, 0)
 
     @mock.patch("rally.api.Verification.verify")
     def test_start_with_wrong_set_name(self, mock_verification_verify):

tests/unit/plugins/openstack/scenarios/tempest/test_tempest.py

@@ -40,7 +40,7 @@ class TempestScenarioTestCase(test.TestCase):
         self.scenario._add_atomic_actions = mock.MagicMock()
 
     def get_tests_launcher_cmd(self, tests):
-        return ("%(venv)s testr run --parallel --subunit %(tests)s "
+        return ("%(venv)s testr run --subunit --parallel %(tests)s "
                 "| tee /dev/null "
                 "| %(venv)s subunit-trace -f -n" %
                 {

tests/unit/test_api.py

@@ -408,7 +408,7 @@ class VerificationAPITestCase(BaseDeploymentTestCase):
 
         self.tempest.is_installed.assert_called_once_with()
         self.tempest.verify.assert_called_once_with(
-            set_name="smoke", regex=None, tests_file=None)
+            set_name="smoke", regex=None, tests_file=None, concur=0)
 
     @mock.patch("rally.api.objects.Deployment.get")
     @mock.patch("rally.api.objects.Verification")
@@ -425,7 +425,7 @@ class VerificationAPITestCase(BaseDeploymentTestCase):
         self.tempest.is_installed.assert_called_once_with()
         self.tempest.install.assert_called_once_with()
         self.tempest.verify.assert_called_once_with(
-            set_name="smoke", regex=None, tests_file=None)
+            set_name="smoke", regex=None, tests_file=None, concur=0)
 
     @mock.patch("os.path.exists", return_value=True)
     @mock.patch("rally.api.objects.Deployment.get")
@@ -441,7 +441,7 @@ class VerificationAPITestCase(BaseDeploymentTestCase):
             self.deployment_uuid, "", None, tests_file, None)
 
         self.tempest.verify.assert_called_once_with(
-            set_name="", regex=None, tests_file=tests_file)
+            set_name="", regex=None, tests_file=tests_file, concur=0)
 
     @mock.patch("rally.common.objects.Deployment.get")
     @mock.patch("rally.api.objects.Verification")

tests/unit/verification/test_tempest.py

@@ -332,15 +332,15 @@ class TempestInstallAndUninstallTestCase(BaseTestCase):
 
 
 class TempestVerifyTestCase(BaseTestCase):
-    def _get_fake_call(self, testr_arg, is_set=True):
+    def _get_fake_call(self, testr_args):
         return (
-            "%(venv)s testr run --parallel --subunit %(testr_arg)s"
+            "%(venv)s testr run --subunit --parallel"
-            " | tee %(tempest_path)s/subunit.stream"
+            " --concurrency 0 %(testr_args)s"
+            " | tee %(log_file)s"
             " | %(venv)s subunit-trace -f -n" % {
                 "venv": self.verifier.venv_wrapper,
-                "testr_arg": ("tempest.api." if is_set
+                "testr_args": testr_args,
-                              else "--load-list ") + testr_arg,
+                "log_file": self.verifier.path("subunit.stream")})
-                "tempest_path": self.verifier.path()})
 
     @mock.patch(TEMPEST_PATH + ".tempest.Tempest.parse_results",
                 return_value=None)
@@ -356,9 +356,9 @@ class TempestVerifyTestCase(BaseTestCase):
             mock_tempest_parse_results):
 
         set_name = "compute"
-        fake_call = self._get_fake_call(set_name)
+        fake_call = self._get_fake_call("tempest.api.%s" % set_name)
 
-        self.verifier.verify(set_name, None, None)
+        self.verifier.verify(set_name, None, None, 0)
 
         self.assertEqual(2, mock_tempest_is_configured.call_count)
         mock_tempest_config.assert_called_once_with(self.verifier.deployment)
@@ -386,9 +386,9 @@ class TempestVerifyTestCase(BaseTestCase):
             mock_tempest_resources_context, mock_subprocess, mock_tempest_env,
             mock_tempest_parse_results):
         set_name = "identity"
-        fake_call = self._get_fake_call(set_name)
+        fake_call = self._get_fake_call("tempest.api.%s" % set_name)
 
-        self.verifier.verify(set_name, None, None)
+        self.verifier.verify(set_name, None, None, 0)
 
         mock_tempest_is_configured.assert_called_once_with()
         self.assertFalse(mock_tempest_config.called)
@@ -414,10 +414,10 @@ class TempestVerifyTestCase(BaseTestCase):
             mock_tempest_resources_context, mock_subprocess, mock_tempest_env,
             mock_tempest_parse_results):
         set_name = "identity"
-        fake_call = self._get_fake_call(set_name)
+        fake_call = self._get_fake_call("tempest.api.%s" % set_name)
         mock_subprocess.side_effect = subprocess.CalledProcessError
 
-        self.verifier.verify(set_name, None, None)
+        self.verifier.verify(set_name, None, None, 0)
 
         mock_tempest_is_configured.assert_called_once_with()
         self.assertFalse(mock_tempest_config.called)
@@ -442,9 +442,9 @@ class TempestVerifyTestCase(BaseTestCase):
             self, mock_tempest_is_configured, mock_tempest_resources_context,
             mock_subprocess, mock_tempest_env, mock_tempest_parse_results):
         tests_file = "/path/to/tests/file"
-        fake_call = self._get_fake_call(tests_file, is_set=False)
+        fake_call = self._get_fake_call("--load-list %s" % tests_file)
 
-        self.verifier.verify("", None, tests_file)
+        self.verifier.verify("", None, tests_file, 0)
         self.verifier.verification.start_verifying.assert_called_once_with("")
 
         mock_subprocess.check_call.assert_called_once_with(