openstack
/
rally
Code Issues Proposed changes

Merge "Re-run failed Tempest tests"

This commit is contained in:
Jenkins 2016-03-02 16:51:04 +00:00 committed by Gerrit Code Review
parent 6923745d0e 7efc0113ef
commit 5d4b5d6e6d
7 changed files with 90 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/rally.bash_completion
View File

@ -55,7 +55,7 @@ _rally()
OPTS["verify_results"]="--uuid --html --json --output-file"
OPTS["verify_show"]="--uuid --sort-by --detailed"
OPTS["verify_showconfig"]="--deployment"
OPTS["verify_start"]="--deployment --set --regex --tests-file --tempest-config --xfails-file --no-use --system-wide --concurrency"
OPTS["verify_start"]="--deployment --set --regex --tests-file --tempest-config --xfails-file --no-use --system-wide --concurrency --failing"
OPTS["verify_uninstall"]="--deployment"
OPTS["verify_use"]="--uuid"

9
rally/api.py
View File

@ -359,9 +359,10 @@ class Verification(object):
@classmethod
def verify(cls, deployment, set_name="", regex=None, tests_file=None,
tempest_config=None, expected_failures=None, system_wide=False,
concur=0):
concur=0, failing=False):
"""Start verification.
:param deployment: UUID or name of a deployment
:param deployment: UUID or name of a deployment
:param set_name: Name of a Tempest test set
:param regex: Regular expression of test
@ -376,8 +377,11 @@ class Verification(object):
env when running the tests
:param concur: How many processes to use to run Tempest tests.
The default value (0) auto-detects CPU count
:param failing: Re-run tests that failed during the last
execution
:returns: Verification object
"""
deployment_uuid = objects.Deployment.get(deployment)["uuid"]
verification = objects.Verification(deployment_uuid=deployment_uuid)
verifier = tempest.Tempest(deployment_uuid,
@ -400,7 +404,8 @@ class Verification(object):
LOG.info("Starting verification of deployment: %s" % deployment_uuid)
verification.set_running()
verifier.verify(set_name=set_name, regex=regex, tests_file=tests_file,
expected_failures=expected_failures, concur=concur)
expected_failures=expected_failures, concur=concur,
failing=failing)
return verification

20
rally/cli/commands/verify.py
View File

@ -75,10 +75,13 @@ class VerifyCommands(object):
required=False,
help="How many processes to use to run Tempest tests. "
"The default value (0) auto-detects your CPU count")
@cliutils.args("--failing", dest="failing", required=False,
help="Re-run the tests that failed in the last execution",
action="store_true")
@envutils.with_default_deployment(cli_arg_name="deployment")
def start(self, deployment=None, set_name="", regex=None,
tests_file=None, tempest_config=None, xfails_file=None,
do_use=True, system_wide=False, concur=0):
do_use=True, system_wide=False, concur=0, failing=False):
"""Start verification (run Tempest tests).
:param deployment: UUID or name of a deployment
@ -95,7 +98,9 @@ class VerifyCommands(object):
env when running the tests
:param concur: How many processes to use to run Tempest tests.
The default value (0) auto-detects CPU count
:param failing: Re-run tests that failed during the last execution
"""
msg = _("Arguments '%s' and '%s' are not compatible. "
"You can use only one of the mentioned arguments.")
if regex and set_name:
@ -108,7 +113,7 @@ class VerifyCommands(object):
print(msg % ("tests_file", "regex"))
return 1
if not (regex or set_name or tests_file):
if not (regex or set_name or tests_file or failing):
set_name = "full"
if set_name and set_name not in AVAILABLE_SETS:
@ -121,6 +126,14 @@ class VerifyCommands(object):
print(_("File '%s' not found.") % tests_file)
return 1
if failing and set_name:
print(msg % ("failing", "set"))
return 1
if failing and tests_file:
print(msg % ("failing", "tests_file"))
return 1
expected_failures = None
if xfails_file:
if os.path.exists(xfails_file):
@ -133,7 +146,8 @@ class VerifyCommands(object):
verification = api.Verification.verify(
deployment, set_name=set_name, regex=regex, tests_file=tests_file,
tempest_config=tempest_config, expected_failures=expected_failures,
system_wide=system_wide, concur=concur)
system_wide=system_wide, concur=concur, failing=failing)
if do_use:
self.use(verification["uuid"])

12
rally/verification/tempest/tempest.py
View File

@ -297,13 +297,16 @@ class Tempest(object):
shutil.rmtree(self.path())
@logging.log_verification_wrapper(LOG.info, _("Run verification."))
def _prepare_and_run(self, set_name, regex, tests_file, concur):
def _prepare_and_run(self, set_name, regex, tests_file, concur, failing):
if not self.is_configured():
self.generate_config_file()
testr_args = "--concurrency %d" % concur
if set_name:
if failing:
testr_args += " --failing"
set_name = "re-run-failed"
elif set_name:
if set_name == "full":
pass
elif set_name in consts.TempestTestsSets:
@ -385,8 +388,9 @@ class Tempest(object):
else:
self.verification.set_failed()
def verify(self, set_name, regex, tests_file, expected_failures, concur):
self._prepare_and_run(set_name, regex, tests_file, concur)
def verify(self, set_name, regex, tests_file, expected_failures, concur,
failing):
self._prepare_and_run(set_name, regex, tests_file, concur, failing)
self._save_results(expected_failures=expected_failures)
def import_results(self, set_name, log_file):

32
tests/unit/cli/commands/test_verify.py
View File

@ -57,8 +57,8 @@ class VerifyCommandsTestCase(test.TestCase):
mock_verification_verify.assert_called_once_with(
deployment_id, set_name="full", regex=None, tests_file=None,
tempest_config=None, expected_failures=None, system_wide=False,
concur=0)
tempest_config=None, expected_failures=None,
system_wide=False, concur=0, failing=False)
@mock.patch("rally.osclients.Clients")
@mock.patch("rally.api.Verification.verify")
@ -76,7 +76,7 @@ class VerifyCommandsTestCase(test.TestCase):
mock_verification_verify.assert_called_once_with(
deployment_id, set_name="full", regex=None, tests_file=None,
tempest_config=tempest_config.name, expected_failures=None,
system_wide=False, concur=0)
system_wide=False, concur=0, failing=False)
tempest_config.close()
@mock.patch("rally.api.Verification.verify")
@ -91,7 +91,7 @@ class VerifyCommandsTestCase(test.TestCase):
mock_verification_verify.assert_called_once_with(
deployment_id, set_name="", regex=None, tests_file=tests_file,
tempest_config=None, expected_failures=None, system_wide=False,
concur=0)
concur=0, failing=False)
@mock.patch("rally.api.Verification.verify")
@mock.patch("six.moves.builtins.open",
@ -107,7 +107,7 @@ class VerifyCommandsTestCase(test.TestCase):
mock_verification_verify.assert_called_once_with(
deployment_id, set_name="full", regex=None, tests_file=None,
tempest_config=None, expected_failures={"test": "reason of fail"},
system_wide=False, concur=0)
system_wide=False, concur=0, failing=False)
@mock.patch("rally.api.Verification.verify")
def test_start_with_wrong_set_name(self, mock_verification_verify):
@ -121,6 +121,28 @@ class VerifyCommandsTestCase(test.TestCase):
consts.TempestTestsAPI)
self.assertFalse(mock_verification_verify.called)
@mock.patch("rally.api.Verification.verify")
def test_start_with_failing_and_set_name(self, mock_verification_verify):
deployment_id = "f2009aae-6ef3-468e-96b2-3c987d584010"
set_name = "some_value"
self.verify.start(set_name=set_name, deployment=deployment_id,
do_use=False, failing=True)
self.assertFalse(mock_verification_verify.called)
@mock.patch("rally.api.Verification.verify")
@mock.patch("os.path.exists", return_value=True)
def test_start_with_failing_and_test_files(self, mock_exists,
mock_verification_verify):
deployment_id = "f2009aae-6ef3-468e-96b2-3c987d584010"
tests_file = "/path/to/tests/file"
self.verify.start(tests_file=tests_file, deployment=deployment_id,
do_use=False, failing=True)
self.assertFalse(mock_verification_verify.called)
@mock.patch("rally.api.Verification.import_results")
def test_import_results(self, mock_verification_import_results):
deployment_id = "fake_uuid"

9
tests/unit/test_api.py
View File

@ -410,7 +410,7 @@ class VerificationAPITestCase(BaseDeploymentTestCase):
self.tempest.is_installed.assert_called_once_with()
self.tempest.verify.assert_called_once_with(
set_name="smoke", regex=None, tests_file=None,
expected_failures=None, concur=0)
expected_failures=None, concur=0, failing=False)
@mock.patch("rally.api.objects.Deployment.get")
@mock.patch("rally.api.objects.Verification")
@ -424,12 +424,11 @@ class VerificationAPITestCase(BaseDeploymentTestCase):
api.Verification.verify(
self.deployment_uuid, set_name="smoke",
regex=None, tests_file=None, tempest_config=None)
self.tempest.is_installed.assert_called_once_with()
self.tempest.install.assert_called_once_with()
self.tempest.verify.assert_called_once_with(
set_name="smoke", regex=None, tests_file=None,
expected_failures=None, concur=0)
expected_failures=None, concur=0, failing=False)
@mock.patch("os.path.exists", return_value=True)
@mock.patch("rally.api.objects.Deployment.get")
@ -443,11 +442,11 @@ class VerificationAPITestCase(BaseDeploymentTestCase):
tests_file = "/path/to/tests/file"
api.Verification.verify(
self.deployment_uuid, set_name="", regex=None,
tests_file=tests_file, tempest_config=None)
tests_file=tests_file, tempest_config=None, failing=False)
self.tempest.verify.assert_called_once_with(
set_name="", regex=None, tests_file=tests_file,
expected_failures=None, concur=0)
expected_failures=None, concur=0, failing=False)
@mock.patch("rally.common.objects.Deployment.get")
@mock.patch("rally.api.objects.Verification")

30
tests/unit/verification/test_tempest.py
View File

@ -359,7 +359,7 @@ class TempestVerifyTestCase(BaseTestCase):
set_name = "compute"
fake_call = self._get_fake_call("tempest.api.%s" % set_name)
self.verifier.verify(set_name, None, None, None, 0)
self.verifier.verify(set_name, None, None, None, 0, False)
self.assertEqual(2, mock_tempest_is_configured.call_count)
mock_tempest_config.assert_called_once_with(self.verifier.deployment)
@ -389,7 +389,7 @@ class TempestVerifyTestCase(BaseTestCase):
set_name = "identity"
fake_call = self._get_fake_call("tempest.api.%s" % set_name)
self.verifier.verify(set_name, None, None, None, 0)
self.verifier.verify(set_name, None, None, None, 0, False)
mock_tempest_is_configured.assert_called_once_with()
self.assertFalse(mock_tempest_config.called)
@ -418,7 +418,7 @@ class TempestVerifyTestCase(BaseTestCase):
fake_call = self._get_fake_call("tempest.api.%s" % set_name)
mock_subprocess.side_effect = subprocess.CalledProcessError
self.verifier.verify(set_name, None, None, None, 0)
self.verifier.verify(set_name, None, None, None, 0, False)
mock_tempest_is_configured.assert_called_once_with()
self.assertFalse(mock_tempest_config.called)
@ -445,7 +445,7 @@ class TempestVerifyTestCase(BaseTestCase):
tests_file = "/path/to/tests/file"
fake_call = self._get_fake_call("--load-list %s" % tests_file)
self.verifier.verify("", None, tests_file, None, 0)
self.verifier.verify("", None, tests_file, None, 0, False)
self.verifier.verification.start_verifying.assert_called_once_with("")
mock_subprocess.check_call.assert_called_once_with(
@ -453,6 +453,28 @@ class TempestVerifyTestCase(BaseTestCase):
shell=True)
mock_tempest_parse_results.assert_called_once_with(None, None)
@mock.patch(TEMPEST_PATH + ".tempest.Tempest.parse_results",
return_value=(None))
@mock.patch(TEMPEST_PATH + ".tempest.Tempest.env")
@mock.patch(TEMPEST_PATH + ".tempest.subprocess")
@mock.patch(TEMPEST_PATH + ".config.TempestResourcesContext")
@mock.patch(TEMPEST_PATH + ".tempest.Tempest.is_configured",
return_value=True)
def test_verify_run_failed_tests_(self, mock_tempest_is_configured,
mock_tempest_resources_context,
mock_subprocess, mock_tempest_env,
mock_tempest_parse_results):
fake_call = self._get_fake_call("--failing")
self.verifier.verify("", None, None, None, 0, True)
self.verifier.verification.start_verifying.assert_called_once_with(
"re-run-failed")
mock_subprocess.check_call.assert_called_once_with(
fake_call, env=mock_tempest_env, cwd=self.verifier.path(),
shell=True)
mock_tempest_parse_results.assert_called_once_with(None, None)
def test_import_results(self):
set_name = "identity"
log_file = "log_file"