openstack/rally
Code Issues Proposed changes

Refactor tempest verification and tests for it

Browse Source 
Class `verification.verifiers.tempest.Tempest` contains next changes:
  Output from installation of virtual environment is not useful for users,
  so visibility of this output was changed and now only in debug mode it
  will be printed.

  The status of verification will be changed to 'FAILED' if something will
  go wrong in subproccess.call

  Two variables contain word 'tempest' in their names. This is redundant,
  so they were renamed:
    Tempest.tempest_base_path => Tempest.base_repo
    Tempest.tempest_path => Tempest._path

  Construction "os.path.join(Tempest.path, some_path)" was moved to method
  `Tempest.path`, since it used in many places.

  Method `Tempest.parse_results` should not be static, because it needed
  inner variable `Tempest.log_file_raw`, so this was fixed.

  "git remote update" is not needed in Tempest installation, so we can
  remove this call and decrease time of installation.

In `rally.cmd.commands.verify.start` command, several issues were fixed:
  First function argument changed to "set_name" instead of "deploy_id".
  Reason: "deploy_id" have default value, so it should be the first in
          arguments. It will simplify command for end-users(launch
          'rally verify start <set_name>' instead of
          'rally verify start --set <set_name>').

  Task commands have cool feature: save task_id in global variables, so
  results cmd can print the last task, without setting it id. This feature
  is ported in verification.

Tests for verification contains a lot of tests, so they are splitted
to separate classes(TempestVerifyTestCase, TempestInstallAndUninstallTestCase
and etc). Also, new tests were added.

Change-Id: I08a52a1e3ceb468ba619049573bcfe642aecbcaf
This commit is contained in:
Andrey Kurilin 2014-10-20 17:11:37 +03:00
parent 2adf9a9c05
commit ff9c4ac151
12 changed files with 347 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
rally/benchmark/context/tempest.py
View File

@ -66,12 +66,12 @@ class Tempest(base.Context):
cmd = ("cd %(tempest_dir)s "
"&& %(venv)s python tempest/stress/tools/cleanup.py" %
{
"tempest_dir": self.verifier.tempest_path,
"tempest_dir": self.verifier.path,
"venv": self.verifier.venv_wrapper})
LOG.debug("Cleanup started by the command: %s" % cmd)
subprocess.check_call(cmd, shell=True, env=self.verifier.env,
cwd=self.verifier.tempest_path)
cwd=self.verifier.path)
except subprocess.CalledProcessError:
LOG.error("Tempest cleanup failed.")

2
rally/cmd/cliutils.py
View File

@ -38,7 +38,7 @@ class CategoryParser(argparse.ArgumentParser):
"""Customized arguments parser
We need this one to override hardcoded behavior.
So, we want to print item's help instead of 'error: too fiew arguments'.
So, we want to print item's help instead of 'error: too few arguments'.
Also, we want not to print positional arguments in help messge.
"""

13
rally/cmd/commands/use.py
View File

@ -113,3 +113,16 @@ class UseCommands(object):
self._ensure_rally_configuration_dir_exists()
db.task_get(task_id)
self._update_attribute_in_global_file('RALLY_TASK', task_id)
@cliutils.args("--uuid", type=str, dest="verification_id", required=False,
help="UUID of the verification")
def verification(self, verification_id):
"""Set active verification.
:param verification_id: a UUID of verification
"""
print('Verification UUID: %s' % verification_id)
self._ensure_rally_configuration_dir_exists()
db.verification_get(verification_id)
self._update_attribute_in_global_file('RALLY_VERIFICATION',
verification_id)

27
rally/cmd/commands/verify.py
View File

@ -22,6 +22,7 @@ import pprint
import six
from rally.cmd import cliutils
from rally.cmd.commands import use
from rally.cmd import envutils
from rally import consts
from rally import db
@ -50,13 +51,15 @@ class VerifyCommands(object):
@cliutils.args("--tempest-config", dest="tempest_config", type=str,
required=False,
help="User specified Tempest config file location")
@cliutils.args("--no-use", action="store_false", dest="do_use",
help="Don't set new task as default for future operations")
@envutils.with_default_deploy_id
def start(self, deploy_id=None, set_name="smoke", regex=None,
tempest_config=None):
def start(self, set_name="smoke", deploy_id=None, regex=None,
tempest_config=None, do_use=False):
"""Start set of tests.
:param deploy_id: a UUID of a deployment
:param set_name: Name of tempest test set
:param deploy_id: a UUID of a deployment
:param regex: Regular expression of test
:param tempest_config: User specified Tempest config file location
"""
@ -66,9 +69,10 @@ class VerifyCommands(object):
if set_name not in consts.TEMPEST_TEST_SETS:
print("Sorry, but there are no desired tempest test set. Please "
"choose from: %s" % ", ".join(consts.TEMPEST_TEST_SETS))
return(1)
api.verify(deploy_id, set_name, regex, tempest_config)
return (1)
verification = api.verify(deploy_id, set_name, regex, tempest_config)
if do_use:
use.UseCommands().verification(verification['uuid'])
def list(self):
"""Display all verifications table, started and finished."""
@ -94,8 +98,9 @@ class VerifyCommands(object):
@cliutils.args('--output-file', type=str, required=False,
dest='output_file',
help='If specified, output will be saved to given file')
def results(self, verification_uuid, output_file=None, output_html=None,
output_json=None, output_pprint=None):
@envutils.with_default_verification_id
def results(self, verification_uuid=None, output_file=None,
output_html=None, output_json=None, output_pprint=None):
"""Get raw results of the verification.
:param verification_uuid: Verification UUID
@ -139,7 +144,8 @@ class VerifyCommands(object):
help='Tests can be sorted by "name" or "duration"')
@cliutils.args('--detailed', dest='detailed', action='store_true',
required=False, help='Prints traceback of failed tests')
def show(self, verification_uuid, sort_by='name', detailed=False):
@envutils.with_default_verification_id
def show(self, verification_uuid=None, sort_by='name', detailed=False):
"""Display results table of the verification."""
try:
@ -191,7 +197,8 @@ class VerifyCommands(object):
help='UUID of a verification')
@cliutils.args('--sort-by', dest='sort_by', type=str, required=False,
help='Tests can be sorted by "name" or "duration"')
def detailed(self, verification_uuid, sort_by='name'):
@envutils.with_default_verification_id
def detailed(self, verification_uuid=None, sort_by='name'):
"""Display results table of verification with detailed errors."""
self.show(verification_uuid, sort_by, True)

9
rally/cmd/envutils.py
View File

@ -21,9 +21,10 @@ from rally import exceptions
from rally import fileutils
from rally.i18n import _
ENV_DEPLOYMENT = 'RALLY_DEPLOYMENT'
ENV_TASK = 'RALLY_TASK'
ENVVARS = [ENV_DEPLOYMENT, ENV_TASK]
ENV_DEPLOYMENT = "RALLY_DEPLOYMENT"
ENV_TASK = "RALLY_TASK"
ENV_VERIFICATION = "RALLY_VERIFICATION"
ENVVARS = [ENV_DEPLOYMENT, ENV_TASK, ENV_VERIFICATION]
MSG_MISSING_ARG = _("Missing argument: --%(arg_name)s")
@ -68,3 +69,5 @@ with_default_deploy_id = default_from_global('deploy_id', ENV_DEPLOYMENT,
"uuid")
with_default_task_id = default_from_global('task_id', ENV_TASK,
"uuid")
with_default_verification_id = default_from_global(
"verification_uuid", ENV_VERIFICATION, "uuid")

2
rally/orchestrator/api.py
View File

@ -177,3 +177,5 @@ def verify(deploy_id, set_name, regex, tempest_config):
verification.set_running()
verifier.verify(set_name=set_name, regex=regex)
return verification

96
rally/verification/verifiers/tempest/tempest.py
View File

@ -35,30 +35,33 @@ class TempestSetupFailure(exceptions.RallyException):
msg_fmt = _("Unable to setup tempest: '%(message)s'")
def check_output(*args, **kwargs):
output = subprocess.check_output(*args, **kwargs)
if LOG.getEffectiveLevel() <= logging.DEBUG:
print(output)
class Tempest(object):
tempest_base_path = os.path.join(os.path.expanduser("~"),
".rally/tempest/base")
base_repo = os.path.join(os.path.expanduser("~"), ".rally/tempest/base")
def __init__(self, deploy_id, verification=None, tempest_config=None):
self.deploy_id = deploy_id
self.tempest_path = os.path.join(os.path.expanduser("~"),
".rally/tempest",
"for-deployment-%s" % deploy_id)
self.config_file = tempest_config or os.path.join(self.tempest_path,
"tempest.conf")
self.log_file_raw = os.path.join(self.tempest_path, "subunit.stream")
self.venv_wrapper = os.path.join(self.tempest_path,
"tools/with_venv.sh")
self._path = os.path.join(os.path.expanduser("~"),
".rally/tempest",
"for-deployment-%s" % deploy_id)
self.config_file = tempest_config or self.path("tempest.conf")
self.log_file_raw = self.path("subunit.stream")
self.venv_wrapper = self.path("tools/with_venv.sh")
self.verification = verification
self._env = None
def _generate_env(self):
env = os.environ.copy()
env["TEMPEST_CONFIG_DIR"] = os.path.split(self.config_file)[0]
env["TEMPEST_CONFIG_DIR"] = os.path.dirname(self.config_file)
env["TEMPEST_CONFIG"] = os.path.basename(self.config_file)
env["OS_TEST_PATH"] = os.path.join(self.tempest_path,
"tempest/test_discover")
env["OS_TEST_PATH"] = self.path("tempest/test_discover")
LOG.debug("Generated environ: %s" % env)
self._env = env
@ -68,18 +71,22 @@ class Tempest(object):
self._generate_env()
return self._env
def path(self, *inner_path):
if inner_path:
return os.path.join(self._path, *inner_path)
return self._path
def _install_venv(self):
if not os.path.isdir(os.path.join(self.tempest_path, '.venv')):
LOG.info('Validating python environment')
path_to_venv = self.path(".venv")
if not os.path.isdir(path_to_venv):
self.validate_env()
LOG.info("No virtual environment found...Install the virtualenv.")
LOG.debug("Virtual environment directory: %s" %
os.path.join(self.tempest_path, ".venv"))
subprocess.check_call("python ./tools/install_venv.py", shell=True,
cwd=self.tempest_path)
subprocess.check_call(
"%s python setup.py install" % self.venv_wrapper,
shell=True, cwd=self.tempest_path)
print("No virtual environment found...Install the virtualenv.")
LOG.debug("Virtual environment directory: %s" % path_to_venv)
check_output("python ./tools/install_venv.py", shell=True,
cwd=self.path())
check_output("%s python setup.py install" % self.venv_wrapper,
shell=True, cwd=self.path())
def is_configured(self):
return os.path.isfile(self.config_file)
@ -98,16 +105,15 @@ class Tempest(object):
LOG.info("Tempest is already configured.")
def _initialize_testr(self):
if not os.path.isdir(os.path.join(self.tempest_path,
".testrepository")):
if not os.path.isdir(self.path(".testrepository")):
msg = _("Test Repository initialization.")
LOG.info(_("Starting: ") + msg)
subprocess.check_call("%s testr init" % self.venv_wrapper,
shell=True, cwd=self.tempest_path)
shell=True, cwd=self.path())
LOG.info(_("Completed: ") + msg)
def is_installed(self):
return os.path.exists(os.path.join(self.tempest_path, ".venv"))
return os.path.exists(self.path(".venv"))
@staticmethod
def _clone():
@ -115,22 +121,19 @@ class Tempest(object):
"This could take a few minutes...")
subprocess.check_call(["git", "clone",
"https://github.com/openstack/tempest",
Tempest.tempest_base_path])
Tempest.base_repo])
def install(self):
if not self.is_installed():
try:
if not os.path.exists(Tempest.tempest_base_path):
if not os.path.exists(Tempest.base_repo):
Tempest._clone()
if not os.path.exists(self.tempest_path):
shutil.copytree(Tempest.tempest_base_path,
self.tempest_path)
if not os.path.exists(self.path()):
shutil.copytree(Tempest.base_repo, self.path())
subprocess.check_call("git checkout master; "
"git remote update; "
"git pull", shell=True,
cwd=os.path.join(self.tempest_path,
"tempest"))
cwd=self.path("tempest"))
self._install_venv()
self._initialize_testr()
except subprocess.CalledProcessError as e:
@ -143,8 +146,8 @@ class Tempest(object):
print("Tempest is already installed")
def uninstall(self):
if os.path.exists(self.tempest_path):
shutil.rmtree(self.tempest_path)
if os.path.exists(self.path()):
shutil.rmtree(self.path())
@utils.log_verification_wrapper(LOG.info, _("Run verification."))
def _prepare_and_run(self, set_name, regex):
@ -167,7 +170,7 @@ class Tempest(object):
try:
self.run(testr_arg)
except subprocess.CalledProcessError:
print("Test set %s has been finished with error. "
print("Test set '%s' has been finished with error. "
"Check log for details" % set_name)
def run(self, testr_arg=None, log_file=None, tempest_conf=None):
@ -197,11 +200,11 @@ class Tempest(object):
{
"venv": self.venv_wrapper,
"arg": testr_arg,
"tempest_path": self.tempest_path,
"tempest_path": self.path(),
"log_file": log_file or self.log_file_raw
})
LOG.debug("Test(s) started by the command: %s" % test_cmd)
subprocess.check_call(test_cmd, cwd=self.tempest_path,
subprocess.check_call(test_cmd, cwd=self.path(),
env=self.env, shell=True)
def discover_tests(self, pattern=""):
@ -211,7 +214,7 @@ class Tempest(object):
"venv": self.venv_wrapper,
"pattern": pattern}
raw_results = subprocess.Popen(
cmd, shell=True, cwd=self.tempest_path, env=self.env,
cmd, shell=True, cwd=self.path(), env=self.env,
stdout=subprocess.PIPE).communicate()[0]
tests = set()
@ -225,10 +228,9 @@ class Tempest(object):
return tests
@staticmethod
def parse_results(log_file_raw):
def parse_results(self, log_file=None):
"""Parse subunit raw log file."""
log_file_raw = log_file or self.log_file_raw
if os.path.isfile(log_file_raw):
data = jsonutils.loads(subunit2json.main(log_file_raw))
return data['total'], data['test_cases']
@ -239,10 +241,12 @@ class Tempest(object):
@utils.log_verification_wrapper(
LOG.info, _("Saving verification results."))
def _save_results(self):
total, test_cases = self.parse_results(self.log_file_raw)
total, test_cases = self.parse_results()
if total and test_cases and self.verification:
self.verification.finish_verification(total=total,
test_cases=test_cases)
else:
self.verification.set_failed()
def validate_env(self):
"""Validate environment parameters required for running tempest.
@ -252,7 +256,7 @@ class Tempest(object):
if sys.version_info < (2, 7):
raise exceptions.IncompatiblePythonVersion(
version=sys.version_info)
version=sys.version_info)
def verify(self, set_name, regex):
self._prepare_and_run(set_name, regex)

4
tests/unit/benchmark/context/test_tempest.py
View File

@ -102,8 +102,8 @@ class TempestContextTestCase(test.TestCase):
mock_sp.check_call.assert_called_once_with(
"cd %s && %s python tempest/stress/tools/cleanup.py" %
(benchmark.verifier.tempest_path, benchmark.verifier.venv_wrapper),
shell=True, cwd=benchmark.verifier.tempest_path,
(benchmark.verifier.path, benchmark.verifier.venv_wrapper),
shell=True, cwd=benchmark.verifier.path,
env=benchmark.verifier.env)
mock_shutil.rmtree.assert_called_once_with("/tmp/path")

22
tests/unit/benchmark/scenarios/tempest/test_tempest.py
View File

@ -44,7 +44,7 @@ class TempestScenarioTestCase(test.TestCase):
"| %(venv)s %(tempest_path)s/tools/colorizer.py" %
{
"venv": self.verifier.venv_wrapper,
"tempest_path": self.verifier.tempest_path,
"tempest_path": self.verifier.path(),
"tests": " ".join(tests)
})
@ -58,7 +58,7 @@ class TempestScenarioTestCase(test.TestCase):
expected_call = self.get_tests_launcher_cmd([fake_test])
mock_sp.check_call.assert_called_once_with(
expected_call, cwd=self.verifier.tempest_path,
expected_call, cwd=self.verifier.path(),
env=self.verifier.env, shell=True)
@mock.patch(TS + ".utils.tempfile")
@ -71,7 +71,7 @@ class TempestScenarioTestCase(test.TestCase):
expected_call = self.get_tests_launcher_cmd([fake_test])
mock_sp.check_call.assert_called_once_with(
expected_call, cwd=self.verifier.tempest_path,
expected_call, cwd=self.verifier.path(),
env=self.verifier.env, shell=True)
@mock.patch(TS + ".utils.tempfile")
@ -83,7 +83,7 @@ class TempestScenarioTestCase(test.TestCase):
expected_call = self.get_tests_launcher_cmd(["tempest.api.network"])
mock_sp.check_call.assert_called_once_with(
expected_call, cwd=self.verifier.tempest_path,
expected_call, cwd=self.verifier.path(),
env=self.verifier.env, shell=True)
@mock.patch(TS + ".utils.tempfile")
@ -95,7 +95,7 @@ class TempestScenarioTestCase(test.TestCase):
expected_call = self.get_tests_launcher_cmd([])
mock_sp.check_call.assert_called_once_with(
expected_call, cwd=self.verifier.tempest_path,
expected_call, cwd=self.verifier.path(),
env=self.verifier.env, shell=True)
@mock.patch(TS + ".utils.tempfile")
@ -107,7 +107,7 @@ class TempestScenarioTestCase(test.TestCase):
expected_call = self.get_tests_launcher_cmd(["smoke"])
mock_sp.check_call.assert_called_once_with(
expected_call, cwd=self.verifier.tempest_path,
expected_call, cwd=self.verifier.path(),
env=self.verifier.env, shell=True)
@mock.patch(TS + ".utils.tempfile")
@ -119,7 +119,7 @@ class TempestScenarioTestCase(test.TestCase):
expected_call = self.get_tests_launcher_cmd([])
mock_sp.check_call.assert_called_once_with(
expected_call, cwd=self.verifier.tempest_path,
expected_call, cwd=self.verifier.path(),
env=self.verifier.env, shell=True)
@mock.patch(TS + ".utils.tempfile")
@ -137,7 +137,7 @@ class TempestScenarioTestCase(test.TestCase):
expected_call = self.get_tests_launcher_cmd([fake_test])
mock_sp.check_call.assert_called_once_with(
expected_call, cwd=self.verifier.tempest_path,
expected_call, cwd=self.verifier.path(),
env=self.verifier.env, shell=True)
@mock.patch(TS + ".utils.tempfile")
@ -149,7 +149,7 @@ class TempestScenarioTestCase(test.TestCase):
expected_call = self.get_tests_launcher_cmd(["tempest.api.network"])
mock_sp.check_call.assert_called_once_with(
expected_call, cwd=self.verifier.tempest_path,
expected_call, cwd=self.verifier.path(),
env=self.verifier.env, shell=True)
@mock.patch(TS + ".utils.tempfile")
@ -162,7 +162,7 @@ class TempestScenarioTestCase(test.TestCase):
expected_call = self.get_tests_launcher_cmd(fake_tests)
mock_sp.check_call.assert_called_once_with(
expected_call, cwd=self.verifier.tempest_path,
expected_call, cwd=self.verifier.path(),
env=self.verifier.env, shell=True)
@mock.patch(TS + ".utils.tempfile")
@ -175,5 +175,5 @@ class TempestScenarioTestCase(test.TestCase):
expected_call = self.get_tests_launcher_cmd([regex])
mock_sp.check_call.assert_called_once_with(
expected_call, cwd=self.verifier.tempest_path,
expected_call, cwd=self.verifier.path(),
env=self.verifier.env, shell=True)

9
tests/unit/cmd/commands/test_verify.py
View File

@ -29,10 +29,6 @@ class VerifyCommandsTestCase(test.TestCase):
def setUp(self):
super(VerifyCommandsTestCase, self).setUp()
self.verify = verify.VerifyCommands()
self.endpoint = {'endpoints': [{'auth_url': 'fake_auth_url',
'username': 'fake_username',
'password': 'fake_password',
'tenant_name': 'fake_tenant_name'}]}
self.image1 = mock.Mock()
self.image1.name = 'cirros-1'
@ -57,7 +53,7 @@ class VerifyCommandsTestCase(test.TestCase):
mock_clients().nova().flavors.list.return_value = [
self.flavor1, self.flavor2]
self.verify.start(deploy_id)
self.verify.start(deploy_id=deploy_id)
default_set_name = 'smoke'
default_regex = None
@ -75,7 +71,8 @@ class VerifyCommandsTestCase(test.TestCase):
mock_clients().nova().flavors.list.return_value = [
self.flavor1, self.flavor2]
tempest_config = tempfile.NamedTemporaryFile()
self.verify.start(deploy_id, tempest_config=tempest_config.name)
self.verify.start(deploy_id=deploy_id,
tempest_config=tempest_config.name)
default_set_name = 'smoke'
default_regex = None

354
tests/unit/verification/verifiers/test_tempest.py
View File

@ -13,7 +13,9 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
import os
import subprocess
import sys
import mock
@ -26,172 +28,120 @@ from rally.verification.verifiers.tempest import tempest
from tests.unit import test
TEMPEST_PATH = 'rally.verification.verifiers.tempest'
TEMPEST_PATH = "rally.verification.verifiers.tempest"
class TempestTestCase(test.TestCase):
class BaseTestCase(test.TestCase):
def setUp(self):
super(TempestTestCase, self).setUp()
super(BaseTestCase, self).setUp()
self.verifier = tempest.Tempest('fake_deploy_id',
verification=mock.MagicMock())
self.verifier.tempest_path = '/tmp'
self.verifier.config_file = '/tmp/tempest.conf'
self.verifier.log_file_raw = '/tmp/subunit.stream'
self.regex = None
self.verifier._path = "/tmp"
self.verifier.config_file = "/tmp/tempest.conf"
self.verifier.log_file_raw = "/tmp/subunit.stream"
@mock.patch('os.path.exists', return_value=True)
class TempestUtilsTestCase(BaseTestCase):
def test_path(self):
self.assertEqual("/tmp", self.verifier.path())
self.assertEqual("/tmp/foo", self.verifier.path("foo"))
self.assertEqual("/tmp/foo/bar", self.verifier.path("foo", "bar"))
@mock.patch("os.path.exists")
def test_is_installed(self, mock_exists):
result = self.verifier.is_installed()
# Check that `is_installed` depends on existence of path
# os.path.exists == True => is_installed == True
mock_exists.return_value = True
self.assertTrue(self.verifier.is_installed())
mock_exists.assert_called_once_with(
os.path.join(self.verifier.tempest_path, '.venv'))
self.assertTrue(result)
# os.path.exists == False => is_installed == False
mock_exists.return_value = False
self.assertFalse(self.verifier.is_installed())
@mock.patch('rally.verification.verifiers.tempest.tempest.subprocess')
def test__clone(self, mock_sp):
self.verifier._clone()
mock_sp.check_call.assert_called_once_with(
['git', 'clone', 'https://github.com/openstack/tempest',
tempest.Tempest.tempest_base_path])
self.assertEqual([mock.call(self.verifier.path(".venv")),
mock.call(self.verifier.path(".venv"))],
mock_exists.call_args_list)
@mock.patch(TEMPEST_PATH + '.tempest.Tempest._initialize_testr')
@mock.patch(TEMPEST_PATH + '.tempest.Tempest._install_venv')
@mock.patch(TEMPEST_PATH + '.tempest.subprocess')
@mock.patch('os.path.exists')
@mock.patch('shutil.copytree')
def test_install(
self, mock_copytree, mock_exists, mock_sp, mock_venv, mock_testr):
mock_exists.side_effect = (False, True, False)
# simulate tempest is clonned but is not installed for current deploy
self.verifier.install()
mock_copytree.assert_called_once_with(
tempest.Tempest.tempest_base_path,
self.verifier.tempest_path)
mock_sp.check_call.assert_called_once_with(
'git checkout master; git remote update; git pull',
cwd=os.path.join(self.verifier.tempest_path, 'tempest'),
shell=True)
@mock.patch('rally.verification.verifiers.tempest.tempest.shutil')
@mock.patch('os.path.exists', return_value=True)
def test_uninstall(self, mock_exists, mock_shutil):
self.verifier.uninstall()
mock_shutil.rmtree.assert_called_once_with(self.verifier.tempest_path)
@mock.patch(TEMPEST_PATH + '.tempest.os.remove')
@mock.patch(TEMPEST_PATH + '.tempest.Tempest._initialize_testr')
@mock.patch(TEMPEST_PATH + '.tempest.Tempest.run')
@mock.patch(TEMPEST_PATH + '.config.TempestConf')
def test_verify(self, mock_conf, mock_run, mock_testr_init, mock_os):
self.verifier.verify("smoke", None)
mock_conf().generate.assert_called_once_with(self.verifier.config_file)
mock_run.assert_called_once_with("smoke")
@mock.patch(TEMPEST_PATH + '.tempest.Tempest.env')
@mock.patch(TEMPEST_PATH + '.tempest.subprocess')
@mock.patch(TEMPEST_PATH + '.config.TempestConf')
@mock.patch(TEMPEST_PATH + '.tempest.Tempest.is_configured',
return_value=False)
def test_verify_complex(self, mock_is_configured, mock_conf,
mock_sp, mock_env):
set_name = "compute"
fake_call = (
"%(venv)s testr run --parallel --subunit tempest.api.%(testr_arg)s"
" | tee %(tempest_path)s/subunit.stream"
" | %(venv)s subunit-2to1"
" | %(venv)s %(tempest_path)s/tools/colorizer.py" % {
"venv": self.verifier.venv_wrapper,
"testr_arg": set_name,
"tempest_path": self.verifier.tempest_path})
self.verifier.verify(set_name, None)
mock_conf.assert_called_once_with(self.verifier.deploy_id)
mock_conf().generate.assert_called_once_with(self.verifier.config_file)
self.verifier.verification.start_verifying.assert_called_once_with(
set_name)
mock_sp.check_call.assert_called_once_with(
fake_call, env=mock_env, cwd=self.verifier.tempest_path,
shell=True)
@mock.patch('os.environ')
def test__generate_env(self, mock_env):
expected_env = {'PATH': '/some/path'}
mock_env.copy.return_value = expected_env.copy()
@mock.patch("os.environ")
def test_env_missed(self, mock_env):
expected_env = {"PATH": "/some/path"}
mock_env.copy.return_value = copy.deepcopy(expected_env)
expected_env.update({
'TEMPEST_CONFIG': 'tempest.conf',
'TEMPEST_CONFIG_DIR': self.verifier.tempest_path,
'OS_TEST_PATH': os.path.join(self.verifier.tempest_path,
'tempest/test_discover')})
"TEMPEST_CONFIG": "tempest.conf",
"TEMPEST_CONFIG_DIR": self.verifier.path(),
"OS_TEST_PATH": self.verifier.path("tempest/test_discover")})
self.assertIsNone(self.verifier._env)
self.verifier._generate_env()
self.assertEqual(expected_env, self.verifier.env)
self.assertTrue(mock_env.copy.called)
self.assertEqual(expected_env, self.verifier._env)
@mock.patch('os.path.isdir', return_value=True)
@mock.patch(TEMPEST_PATH + '.tempest.subprocess')
@mock.patch("os.environ")
def test_env_loaded(self, mock_env):
self.verifier._env = {"foo": "bar"}
self.verifier.env
self.assertFalse(mock_env.copy.called)
@mock.patch("os.path.isdir", return_value=True)
@mock.patch(TEMPEST_PATH + ".tempest.subprocess")
@testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version")
def test__venv_install_when_venv_exists(self, mock_sp, mock_isdir):
self.verifier._install_venv()
mock_isdir.assert_called_once_with(
os.path.join(self.verifier.tempest_path, '.venv'))
self.assertFalse(mock_sp.called)
mock_isdir.assert_called_once_with(self.verifier.path(".venv"))
self.assertFalse(mock_sp.check_output.called)
@mock.patch('os.path.isdir', return_value=False)
@mock.patch(TEMPEST_PATH + '.tempest.subprocess.check_call')
@mock.patch("os.path.isdir", return_value=False)
@mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_output")
@testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version")
def test__venv_install_when_venv_not_exist(self, mock_sp, mock_isdir):
self.verifier._install_venv()
mock_isdir.assert_called_once_with(
os.path.join(self.verifier.tempest_path, '.venv'))
mock_isdir.assert_called_once_with(self.verifier.path(".venv"))
mock_sp.assert_has_calls([
mock.call('python ./tools/install_venv.py', shell=True,
cwd=self.verifier.tempest_path),
mock.call('%s python setup.py install' %
mock.call("python ./tools/install_venv.py", shell=True,
cwd=self.verifier.path()),
mock.call("%s python setup.py install" %
self.verifier.venv_wrapper, shell=True,
cwd=self.verifier.tempest_path)])
cwd=self.verifier.path())])
@mock.patch('os.path.isdir', return_value=False)
@mock.patch("os.path.isdir", return_value=False)
@testtools.skipIf(sys.version_info >= (2, 7),
"Incompatible Python Version")
def test__venv_install_for_py26_fails(self, mock_isdir):
self.assertRaises(exceptions.IncompatiblePythonVersion,
self.verifier._install_venv)
mock_isdir.assert_called_once_with(
os.path.join(self.verifier.tempest_path, '.venv'))
mock_isdir.assert_called_once_with(self.verifier.path(".venv"))
@mock.patch('os.path.isdir', return_value=True)
@mock.patch(TEMPEST_PATH + '.tempest.subprocess')
@mock.patch("os.path.isdir", return_value=True)
@mock.patch(TEMPEST_PATH + ".tempest.subprocess")
def test__initialize_testr_when_testr_already_initialized(
self, mock_sp, mock_isdir):
self.verifier._initialize_testr()
mock_isdir.assert_called_once_with(
os.path.join(self.verifier.tempest_path, '.testrepository'))
self.verifier.path(".testrepository"))
self.assertFalse(mock_sp.called)
@mock.patch('os.path.isdir', return_value=False)
@mock.patch(TEMPEST_PATH + '.tempest.subprocess.check_call')
@mock.patch("os.path.isdir", return_value=False)
@mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_call")
def test__initialize_testr_when_testr_not_initialized(
self, mock_sp, mock_isdir):
self.verifier._initialize_testr()
mock_isdir.assert_called_once_with(
os.path.join(self.verifier.tempest_path, '.testrepository'))
self.verifier.path(".testrepository"))
mock_sp.assert_called_once_with(
'%s testr init' % self.verifier.venv_wrapper, shell=True,
cwd=self.verifier.tempest_path)
cwd=self.verifier.path())
@mock.patch.object(subunit2json, 'main')
@mock.patch('os.path.isfile', return_value=False)
def test__save_results_without_log_file(self, mock_isfile, mock_parse):
self.verifier._save_results()
mock_isfile.assert_called_once_with(self.verifier.log_file_raw)
self.assertEqual(0, mock_parse.call_count)
@mock.patch('os.path.isfile', return_value=True)
@ -200,12 +150,184 @@ class TempestTestCase(test.TestCase):
data = {'total': True, 'test_cases': True}
mock_main.return_value = jsonutils.dumps(data)
self.verifier.log_file_raw = os.path.join(
os.path.dirname(__file__),
'subunit.stream')
os.path.dirname(__file__), 'subunit.stream')
self.verifier._save_results()
mock_isfile.assert_called_once_with(self.verifier.log_file_raw)
mock_main.assert_called_once_with(
self.verifier.log_file_raw)
self.assertEqual(
1, self.verifier.verification.finish_verification.call_count)
verification = self.verifier.verification
verification.finish_verification.assert_called_once_with(**data)
class TempestInstallAndUninstallTestCase(BaseTestCase):
@mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_call")
def test__clone_successful(self, mock_sp):
self.verifier._clone()
mock_sp.assert_called_once_with(
['git', 'clone', 'https://github.com/openstack/tempest',
tempest.Tempest.base_repo])
@mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_call")
def test__clone_failed(self, mock_sp):
# Check that `subprocess.CalledProcessError` is not handled by `_clone`
mock_sp.side_effect = subprocess.CalledProcessError(0, None)
self.assertRaises(subprocess.CalledProcessError, self.verifier._clone)
mock_sp.assert_called_once_with(
['git', 'clone', 'https://github.com/openstack/tempest',
tempest.Tempest.base_repo])
@mock.patch(TEMPEST_PATH + ".tempest.Tempest._initialize_testr")
@mock.patch(TEMPEST_PATH + ".tempest.Tempest._install_venv")
@mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_call")
@mock.patch("shutil.copytree")
@mock.patch(TEMPEST_PATH + ".tempest.Tempest._clone")
@mock.patch("os.path.exists", return_value=False)
def test_install_successful(self, mock_exists, mock_clone, mock_copytree,
mock_sp, mock_install_venv, mock_testr_init):
self.verifier.install()
self.assertEqual([mock.call(self.verifier.path(".venv")),
mock.call(self.verifier.base_repo),
mock.call(self.verifier.path())],
mock_exists.call_args_list)
mock_clone.assert_called_once_with()
mock_copytree.assert_called_once_with(
self.verifier.base_repo,
self.verifier.path())
mock_sp.assert_called_once_with(
"git checkout master; git pull",
cwd=self.verifier.path("tempest"),
shell=True)
mock_install_venv.assert_called_once_with()
mock_testr_init.assert_called_once_with()
@mock.patch(TEMPEST_PATH + ".tempest.Tempest.uninstall")
@mock.patch(TEMPEST_PATH + ".tempest.Tempest._initialize_testr")
@mock.patch(TEMPEST_PATH + ".tempest.Tempest._install_venv")
@mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_call")
@mock.patch("shutil.copytree")
@mock.patch(TEMPEST_PATH + ".tempest.Tempest._clone")
@mock.patch("os.path.exists", return_value=False)
def test_install_failed(self, mock_exists, mock_clone, mock_copytree,
mock_sp, mock_install_venv, mock_testr_init,
mock_uninstall):
mock_sp.side_effect = subprocess.CalledProcessError(0, None)
self.assertRaises(tempest.TempestSetupFailure, self.verifier.install)
self.assertEqual([mock.call(self.verifier.path(".venv")),
mock.call(self.verifier.base_repo),
mock.call(self.verifier.path())],
mock_exists.call_args_list)
mock_clone.assert_called_once_with()
mock_copytree.assert_called_once_with(
self.verifier.base_repo,
self.verifier.path())
mock_sp.assert_called_once_with(
"git checkout master; git pull",
cwd=self.verifier.path("tempest"),
shell=True)
self.assertFalse(mock_install_venv.called)
self.assertFalse(mock_testr_init.called)
mock_uninstall.assert_called_once_with()
@mock.patch("shutil.rmtree")
@mock.patch("os.path.exists", return_value=True)
def test_uninstall(self, mock_exists, mock_shutil):
self.verifier.uninstall()
mock_exists.assert_called_once_with(self.verifier.path())
mock_shutil.assert_called_once_with(self.verifier.path())
class TempestVerifyTestCase(BaseTestCase):
def _get_fake_call(self, testr_arg):
return (
"%(venv)s testr run --parallel --subunit tempest.api.%(testr_arg)s"
" | tee %(tempest_path)s/subunit.stream"
" | %(venv)s subunit-2to1"
" | %(venv)s %(tempest_path)s/tools/colorizer.py" % {
"venv": self.verifier.venv_wrapper,
"testr_arg": testr_arg,
"tempest_path": self.verifier.path()})
@mock.patch(TEMPEST_PATH + ".tempest.Tempest.parse_results",
return_value=(None, None))
@mock.patch(TEMPEST_PATH + ".tempest.Tempest.env")
@mock.patch(TEMPEST_PATH + ".tempest.subprocess")
@mock.patch(TEMPEST_PATH + ".config.TempestConf")
@mock.patch(TEMPEST_PATH + ".tempest.Tempest.is_configured",
return_value=False)
def test_verify_not_configured(self, mock_is_configured, mock_conf,
mock_sp, mock_env, mock_parse_results):
set_name = "compute"
fake_call = self._get_fake_call(set_name)
self.verifier.verify(set_name, None)
self.assertEqual(2, mock_is_configured.call_count)
mock_conf.assert_called_once_with(self.verifier.deploy_id)
mock_conf().generate.assert_called_once_with(self.verifier.config_file)
self.verifier.verification.start_verifying.assert_called_once_with(
set_name)
mock_sp.check_call.assert_called_once_with(
fake_call, env=mock_env, cwd=self.verifier.path(),
shell=True)
mock_parse_results.assert_called_once_with()
@mock.patch(TEMPEST_PATH + ".tempest.Tempest.parse_results",
return_value=(None, None))
@mock.patch(TEMPEST_PATH + ".tempest.Tempest.env")
@mock.patch(TEMPEST_PATH + ".tempest.subprocess")
@mock.patch(TEMPEST_PATH + ".config.TempestConf")
@mock.patch(TEMPEST_PATH + ".tempest.Tempest.is_configured",
return_value=True)
def test_verify_when_tempest_configured(self, mock_is_configured,
mock_conf, mock_sp, mock_env,
mock_parse_results):
set_name = "identity"
fake_call = self._get_fake_call(set_name)
self.verifier.verify(set_name, None)
mock_is_configured.assert_called_once_with()
self.assertFalse(mock_conf.called)
self.assertFalse(mock_conf().generate.called)
self.verifier.verification.start_verifying.assert_called_once_with(
set_name)
mock_sp.check_call.assert_called_once_with(
fake_call, env=mock_env, cwd=self.verifier.path(),
shell=True)
mock_parse_results.assert_called_once_with()
@mock.patch(TEMPEST_PATH + ".tempest.Tempest.parse_results",
return_value=(None, None))
@mock.patch(TEMPEST_PATH + ".tempest.Tempest.env")
@mock.patch(TEMPEST_PATH + ".tempest.subprocess")
@mock.patch(TEMPEST_PATH + ".config.TempestConf")
@mock.patch(TEMPEST_PATH + ".tempest.Tempest.is_configured",
return_value=True)
def test_verify_failed_and_tempest_is_configured(
self, mock_is_configured, mock_conf, mock_sp, mock_env,
mock_parse_results):
set_name = "identity"
fake_call = self._get_fake_call(set_name)
mock_sp.side_effect = subprocess.CalledProcessError
self.verifier.verify(set_name, None)
mock_is_configured.assert_called_once_with()
self.assertFalse(mock_conf.called)
self.assertFalse(mock_conf().generate.called)
self.verifier.verification.start_verifying.assert_called_once_with(
set_name)
mock_sp.check_call.assert_called_once_with(
fake_call, env=mock_env, cwd=self.verifier.path(),
shell=True)
self.assertTrue(mock_parse_results.called)
self.verifier.verification.set_failed.assert_called_once_with()

3
tools/rally.bash_completion
View File

@ -9,6 +9,7 @@ _rally()
OPTS["info_find"]="--query"
OPTS["use_deployment"]="--uuid --name"
OPTS["use_task"]="--uuid"
OPTS["use_verification"]="--uuid"
OPTS["task_abort"]="--uuid"
OPTS["task_delete"]="--force --uuid"
OPTS["task_detailed"]="--uuid --iterations-data"
@ -29,7 +30,7 @@ _rally()
OPTS["verify_list"]=""
OPTS["verify_results"]="--uuid --html --json --pprint --output-file"
OPTS["verify_show"]="--uuid --sort-by --detailed"
OPTS["verify_start"]="--deploy-id --set --regex --tempest-config"
OPTS["verify_start"]="--deploy-id --set --regex --tempest-config --no-use"
OPTS["deployment_check"]="--uuid"
OPTS["deployment_config"]="--uuid --json --pprint"
OPTS["deployment_create"]="--name --fromenv --filename --no-use"