Adds saver for results from rally verify

Better integration with tempest can be achieved by analyzing output
of testr. This patch provides parsing testr output and saves results
into database.

Blueprint tempest-verification
Closes-Bug: #1302475

Change-Id: I110bc115bd597df54499ac295d836a87411c4c80

rally/db/api.py

@@ -246,3 +246,73 @@ def resource_delete(id):
              does not exist.
     """
     return IMPL.resource_delete(id)
+
+
+def verification_create(deployment_uuid):
+    """Create Verification record in DB.
+
+    :param deployment_uuid: UUID of the deployment.
+    :returns: a dict with verification data.
+    """
+    return IMPL.verification_create(deployment_uuid)
+
+
+def verification_get(verification_uuid):
+    """Returns verification by UUID.
+
+    :param id: UUID of the verification.
+    :raises: :class:`rally.exceptions.NotFoundException` if verification
+             does not exist.
+    :returns: a dict with verification data.
+    """
+    return IMPL.verification_get(verification_uuid)
+
+
+def verification_delete(verification_uuid):
+    """Delete verification.
+
+    :param verification_uuid: UUID of the verification.
+    :raises: :class:`rally.exceptions.NotFoundException` if verification
+             does not exist.
+    """
+    return IMPL.verification_delete(verification_uuid)
+
+
+def verification_update(uuid, values):
+    """Update verification by values.
+
+    :param uuid: UUID of the verification.
+    :param values: dict with record values.
+    :raises: :class:`rally.exceptions.NotFoundException` if verification
+             does not exist.
+    :returns: new updated task dict with data on the task.
+    """
+    return IMPL.verification_update(uuid, values)
+
+
+def verification_list(status=None):
+    """Get a list of verifications.
+
+    :param status: Verification status to filter the returned list on.
+    :returns: A list of dicts with data on the verifications.
+    """
+    return IMPL.verification_list(status=status)
+
+
+def verification_result_get(verification_uuid):
+    """Get dict of verification results.
+
+    :param verification_uuid: string with UUID of Verification instance.
+    :returns: dict instance of VerificationResult.
+    """
+    return IMPL.verification_result_get(verification_uuid)
+
+
+def verification_result_create(verification_uuid, values):
+    """Append result record to verification.
+
+    :param verification_uuid: string with UUID of Verification instance.
+    :param values: dict with record values.
+    :returns: TaskResult instance appended.
+    """
+    return IMPL.verification_result_create(verification_uuid, values)

rally/db/sqlalchemy/api.py

@@ -257,3 +257,61 @@ def resource_delete(id):
                 delete(synchronize_session=False)
     if not count:
         raise exceptions.ResourceNotFound(id=id)
+
+
+def verification_create(deployment_uuid):
+    verification = models.Verification()
+    verification.update({"deployment_uuid": deployment_uuid})
+    verification.save()
+    return verification
+
+
+def verification_get(verification_uuid, session=None):
+    verification = model_query(models.Verification, session=session).\
+        filter_by(uuid=verification_uuid).first()
+    if not verification:
+        raise exceptions.NotFoundException(
+            "Can't find any verification with following UUID '%s'." %
+            verification_uuid)
+    return verification
+
+
+def verification_update(verification_uuid, values):
+    session = get_session()
+    with session.begin():
+        verification = verification_get(verification_uuid, session=session)
+        verification.update(values)
+    return verification
+
+
+def verification_list(status=None):
+    query = model_query(models.Verification)
+    if status is not None:
+        query = query.filter_by(status=status)
+    return query.all()
+
+
+def verification_delete(verification_uuid):
+    count = model_query(models.Verification).filter_by(id=verification_uuid).\
+        delete(synchronize_session=False)
+    if not count:
+        raise exceptions.NotFoundException(
+            "Can't find any verification with following UUID '%s'." %
+            verification_uuid)
+
+
+def verification_result_create(verification_uuid, data):
+    result = models.VerificationResult()
+    result.update({"verification_uuid": verification_uuid,
+                   "data": data})
+    result.save()
+    return result
+
+
+def verification_result_get(verification_uuid):
+    result = model_query(models.VerificationResult).\
+        filter_by(verification_uuid=verification_uuid).first()
+    if not result:
+        raise exceptions.NotFoundException(
+            "No results for following UUID '%s'." % verification_uuid)
+    return result

rally/db/sqlalchemy/models.py

@@ -132,7 +132,7 @@ class Resource(BASE, RallyBase):
 
 
 class Task(BASE, RallyBase):
-    """Represents a Benchamrk task."""
+    """Represents a Benchmark task."""
     __tablename__ = "tasks"
     __table_args__ = (
         sa.Index("task_uuid", "uuid", unique=True),
@@ -177,6 +177,47 @@ class TaskResult(BASE, RallyBase):
                                primaryjoin='TaskResult.task_uuid == Task.uuid')
 
 
+class Verification(BASE, RallyBase):
+    """Represents a verifier result."""
+
+    __tablename__ = "verifications"
+    __table_args__ = (
+        sa.Index("verification_uuid", "uuid", unique=True),
+    )
+
+    id = sa.Column(sa.Integer, primary_key=True, autoincrement=True)
+    uuid = sa.Column(sa.String(36), default=UUID, nullable=False)
+
+    deployment_uuid = sa.Column(
+        sa.String(36),
+        sa.ForeignKey(Deployment.uuid),
+        nullable=False,
+    )
+
+    status = sa.Column(sa.Enum(*list(consts.TaskStatus),
+                       name="enum_tasks_status"),
+                       default=consts.TaskStatus.INIT,
+                       nullable=False)
+    set_name = sa.Column(sa.String(20))
+
+    tests = sa.Column(sa.Integer, default=0)
+    errors = sa.Column(sa.Integer, default=0)
+    failures = sa.Column(sa.Integer, default=0)
+    time = sa.Column(sa.Float, default=0.0)
+
+
+class VerificationResult(BASE, RallyBase):
+    __tablename__ = "verification_results"
+    __table_args__ = ()
+
+    id = sa.Column(sa.Integer, primary_key=True, autoincrement=True)
+
+    verification_uuid = sa.Column(sa.String(36),
+                                  sa.ForeignKey('verifications.uuid'))
+
+    data = sa.Column(sa_types.MutableJSONEncodedDict, nullable=False)
+
+
 def create_db():
     from rally.db.sqlalchemy import api as sa_api
 

rally/objects/__init__.py

@@ -17,3 +17,4 @@
 from rally.objects.deploy import Deployment  # noqa
 from rally.objects.endpoint import Endpoint  # noqa
 from rally.objects.task import Task  # noqa
+from rally.objects.verification import Verification  # noqa

rally/objects/verification.py

@@ -0,0 +1,71 @@
+# Copyright 2014: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from rally import consts
+from rally import db
+from rally import exceptions
+
+
+class Verification(object):
+    """Represents results of verification."""
+
+    def __init__(self, db_object=None, deployment_uuid=None):
+        if db_object:
+            self.db_object = db_object
+        else:
+            self.db_object = db.verification_create(deployment_uuid)
+
+    def __getattr__(self, item):
+        return self.db_object[item]
+
+    def __getitem__(self, key):
+        return self.db_object[key]
+
+    @classmethod
+    def get(cls, uuid):
+        return cls(db.verification_get(uuid))
+
+    def delete(self):
+        db.verification_delete(self.db_object['uuid'])
+
+    def _update(self, **values):
+        self.db_object = db.verification_update(self.uuid, values)
+
+    def update_status(self, status):
+        self._update(status=status)
+
+    def start_verifying(self, set_name):
+        self._update(status=consts.TaskStatus.VERIFYING, set_name=set_name)
+
+    def set_failed(self):
+        self.update_status(consts.TaskStatus.FAILED)
+
+    def set_running(self):
+        self.update_status(consts.TaskStatus.RUNNING)
+
+    def finish_verification(self, total, test_cases):
+        # update verification db object
+        self._update(status=consts.TaskStatus.FINISHED, **total)
+
+        # create db object for results
+        data = total.copy()
+        data['test_cases'] = test_cases
+        db.verification_result_create(self.uuid, data)
+
+    def get_results(self):
+        try:
+            return db.verification_result_get(self.uuid)
+        except exceptions.NotFoundException:
+            return None

rally/orchestrator/api.py

@@ -13,7 +13,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-
 import re
 
 from rally.benchmark import engine
@@ -22,8 +21,11 @@ from rally import db
 from rally import deploy
 from rally import exceptions
 from rally import objects
+from rally.openstack.common import log as logging
 from rally.verification.verifiers.tempest import tempest
 
+LOG = logging.getLogger(__name__)
+
 
 def create_deploy(config, name):
     """Create a deployment.
@@ -142,12 +144,13 @@ def verify(deploy_id, image_id, alt_image_id, flavor_id, alt_flavor_id,
     :param alt_flavor_id: Valid secondary flavor to be used in tests.
     :param set_name: Valid name of tempest test set.
     """
-    verifier = tempest.Tempest(deploy_id)
+    verification = objects.Verification(deployment_uuid=deploy_id)
+    verifier = tempest.Tempest(deploy_id, verification)
     if not verifier.is_installed():
         print("Tempest is not installed for specified deployment. "
               "Please use 'rally-manage tempest install'")
         return
-    print("Starting verification of deployment: %s" % deploy_id)
+    LOG.info("Starting verification of deployment: %s" % deploy_id)
 
     endpoints = db.deployment_get(deploy_id)['endpoints']
     endpoint = endpoints[0]
@@ -167,4 +170,5 @@ def verify(deploy_id, image_id, alt_image_id, flavor_id, alt_flavor_id,
                      ('uri_v3', re.sub('/v2.0', '/v3', endpoint['auth_url']))])
                  )
 
+    verification.set_running()
     verifier.verify(set_name=set_name, regex=regex, options=conf_opts)

rally/utils.py

@@ -159,3 +159,7 @@ def log_task_wrapper(log, msg, **kw):
 
 def log_deploy_wrapper(log, msg, **kw):
     return _log_wrapper('deployment', log, msg, **kw)
+
+
+def log_verification_wrapper(log, msg, **kw):
+    return _log_wrapper('verification', log, msg, **kw)

rally/verification/verifiers/tempest/tempest.py

@@ -19,9 +19,13 @@ import os
 import shutil
 import subprocess
 import tempfile
+from xml.dom import minidom as md
 
 from six.moves import configparser
 
+from rally.openstack.common.gettextutils import _
+from rally import utils
+
 LOG = logging.getLogger(__name__)
 
 
@@ -30,14 +34,16 @@ class Tempest(object):
     tempest_base_path = os.path.join(os.path.expanduser("~"),
                                      '.rally/tempest/base')
 
-    def __init__(self, deploy_id):
+    def __init__(self, deploy_id, verification=None):
         self.lock_path = tempfile.mkdtemp()
         self.tempest_path = os.path.join(os.path.expanduser("~"),
                                          '.rally/tempest',
                                          'for-deployment-%s' % deploy_id)
         self.config_file = os.path.join(self.tempest_path, 'tempest.conf')
+        self.log_file = os.path.join(self.tempest_path, 'testr_log.xml')
         self._venv_wrapper = os.path.join(self.tempest_path,
                                           'tools/with_venv.sh')
+        self.verification = verification
 
     def _generate_config(self, options):
         conf = configparser.ConfigParser()
@@ -64,14 +70,30 @@ class Tempest(object):
         LOG.debug('Generated environ: %s' % env)
         return env
 
-    def _check_venv_existence(self):
+    def _install_venv(self):
         if not os.path.isdir(os.path.join(self.tempest_path, '.venv')):
             LOG.info('No virtual environment found...Install the virtualenv.')
             LOG.debug('Virtual environment directory: %s' %
                       os.path.join(self.tempest_path, '.venv'))
-            subprocess.call('python ./tools/install_venv.py', shell=True,
-                            cwd=self.tempest_path)
+            subprocess.check_call('python ./tools/install_venv.py', shell=True,
+                                  cwd=self.tempest_path)
+            # NOTE(akurilin): junitxml is required for subunit2junitxml filter.
+            # This library not in openstack/requirements, so we must install it
+            # by this way.
+            subprocess.check_call(
+                '%s pip install junitxml' % self._venv_wrapper,
+                shell=True, cwd=self.tempest_path)
 
+    @utils.log_verification_wrapper(LOG.info,
+                                    _('Check existence of configuration file'))
+    def _check_config_existence(self, options):
+        LOG.debug("Tempest config file: %s " % self.config_file)
+        if not os.path.isfile(self.config_file):
+            conf = self._generate_config(options)
+            self._write_config(conf)
+
+    @utils.log_verification_wrapper(
+        LOG.info, _('Check initialization of test repository.'))
     def _check_testr_initialization(self):
         if not os.path.isdir(os.path.join(self.tempest_path,
                                           '.testrepository')):
@@ -79,46 +101,73 @@ class Tempest(object):
                             cwd=self.tempest_path)
 
     def is_installed(self):
-        return os.path.exists(self.tempest_path)
+        return os.path.exists(os.path.join(self.tempest_path, '.venv'))
 
     @staticmethod
     def _clone():
-        subprocess.call(['git', 'clone', 'git://github.com/openstack/tempest',
-                        Tempest.tempest_base_path])
+        print('Please wait while tempest is being cloned. '
+              'This could take a few minutes...')
+        subprocess.check_call(['git', 'clone',
+                               'git://github.com/openstack/tempest',
+                               Tempest.tempest_base_path])
 
     def install(self):
-        if not os.path.exists(Tempest.tempest_base_path):
-            Tempest._clone()
-        if os.path.exists(self.tempest_path):
-            print('Tempest is already installed')
+        if not self.is_installed():
+            try:
+                if not os.path.exists(Tempest.tempest_base_path):
+                    Tempest._clone()
+
+                if not os.path.exists(self.tempest_path):
+                    shutil.copytree(Tempest.tempest_base_path,
+                                    self.tempest_path)
+                    subprocess.check_call('git checkout master; '
+                                          'git remote update; '
+                                          'git pull', shell=True,
+                                          cwd=os.path.join(self.tempest_path,
+                                                           'tempest'))
+                self._install_venv()
+            except subprocess.CalledProcessError:
+                print ('Tempest installation failed.')
+                return 1
+            else:
+                print('Tempest has been successfully installed!')
         else:
-            shutil.copytree(Tempest.tempest_base_path, self.tempest_path)
-            subprocess.Popen('git checkout master; git remote update; '
-                             'git pull', shell=True,
-                             cwd=os.path.join(self.tempest_path,
-                                              'tempest')).communicate()
-            print('Tempest has been successfully installed!')
+            print('Tempest is already installed')
 
     def uninstall(self):
         if os.path.exists(self.tempest_path):
             shutil.rmtree(self.tempest_path)
 
-    def _run(self, set_name, regex):
-        if set_name == 'full':
-            set_path = ''
-        elif set_name == 'smoke':
-            set_path = 'smoke'
-        else:
-            set_path = 'tempest.api.%s' % set_name
-        regex = regex if regex else ''
+    @utils.log_verification_wrapper(LOG.info, _('Run verification.'))
+    def _prepare_and_run(self, set_name, regex, options):
+        self._check_config_existence(options)
+        self._check_testr_initialization()
 
-        testr_runner = '%(venv)s testr run --parallel --subunit ' \
-                       '%(set_path)s %(regex)s | %(venv)s subunit-2to1 ' \
-                       '| %(venv)s %(tempest_path)s/tools/colorizer.py' % {
-                           'venv': self._venv_wrapper,
-                           'set_path': set_path,
-                           'regex': regex,
-                           'tempest_path': self.tempest_path}
+        if set_name == 'full':
+            testr_arg = ''
+        elif set_name == 'smoke':
+            testr_arg = 'smoke'
+        else:
+            testr_arg = 'tempest.api.%s' % set_name
+
+        if regex:
+            testr_arg += ' %s' % regex
+
+        self.verification.start_verifying(set_name)
+        self._run(testr_arg)
+
+    def _run(self, testr_arg):
+        testr_runner = (
+            '%(venv)s testr run --parallel --subunit %(arg)s '
+            '| %(venv)s subunit2junitxml --forward --output-to=%(log_file)s '
+            '| %(venv)s subunit-2to1 '
+            '| %(venv)s %(tempest_path)s/tools/colorizer.py' %
+            {
+                'venv': self._venv_wrapper,
+                'arg': testr_arg,
+                'tempest_path': self.tempest_path,
+                'log_file': self.log_file
+            })
         try:
             LOG.debug('testr started by the command: %s' % testr_runner)
             subprocess.check_call(testr_runner,
@@ -126,18 +175,48 @@ class Tempest(object):
                                   env=self._generate_env(), shell=True)
         except subprocess.CalledProcessError:
             print('Test set %s has been finished with error. '
-                  'Check log for details' % set_name)
+                  'Check log for details' % testr_arg)
         finally:
             shutil.rmtree(self.lock_path)
 
-        #TODO(miarmak) Change log_file and parse it
+    @utils.log_verification_wrapper(
+        LOG.info, _('Saving verification results.'))
+    def _save_results(self):
+        if os.path.isfile(self.log_file):
+            dom = md.parse(self.log_file).getElementsByTagName('testsuite')[0]
+
+            total = {
+                'tests': int(dom.getAttribute('tests')),
+                'errors': int(dom.getAttribute('errors')),
+                'failures': int(dom.getAttribute('failures')),
+                'time': float(dom.getAttribute('time')),
+            }
+
+            test_cases = {}
+            for test_elem in dom.getElementsByTagName('testcase'):
+                if test_elem.getAttribute('name') == 'process-returncode':
+                    total['failures'] -= 1
+                else:
+                    test = {
+                        'name': ".".join((test_elem.getAttribute('classname'),
+                                          test_elem.getAttribute('name'))),
+                        'time': float(test_elem.getAttribute('time'))
+                    }
+
+                    failure = test_elem.getElementsByTagName('failure')
+                    if failure:
+                        test['status'] = 'FAIL'
+                        test['failure'] = {
+                            'type': failure[0].getAttribute('type'),
+                            'log': failure[0].firstChild.nodeValue}
+                    else:
+                        test['status'] = 'OK'
+                    test_cases[test['name']] = test
+            self.verification.finish_verification(total=total,
+                                                  test_cases=test_cases)
+        else:
+            LOG.error('XML-log file not found.')
 
     def verify(self, set_name, regex, options):
-        if not os.path.isfile(self.config_file):
-            conf = self._generate_config(options)
-            self._write_config(conf)
-        LOG.debug("Tempest config file: %s " % self.config_file)
-        self._check_venv_existence()
-        self._check_testr_initialization()
-
-        self._run(set_name, regex)
+        self._prepare_and_run(set_name, regex, options)
+        self._save_results()

tests/db/test_api.py

@@ -391,3 +391,22 @@ class ResourceTestCase(test.DBTestCase):
                                         type='two')
         self.assertEqual(len(resources), 1)
         self.assertEqual(res_two['id'], resources[0]['id'])
+
+
+class VerificationTestCase(test.DBTestCase):
+    def setUp(self):
+        super(VerificationTestCase, self).setUp()
+        self.deploy = db.deployment_create({})
+
+    def _create_verification(self):
+        deployment_uuid = self.deploy['uuid']
+        return db.verification_create(deployment_uuid)
+
+    def test_creation_of_verification(self):
+        verification = self._create_verification()
+        db_verification = db.verification_get(verification['uuid'])
+
+        self.assertEqual(verification['tests'], db_verification['tests'])
+        self.assertEqual(verification['time'], db_verification['time'])
+        self.assertEqual(verification['errors'], db_verification['errors'])
+        self.assertEqual(verification['failures'], db_verification['failures'])

tests/objects/test_verification.py

@@ -0,0 +1,90 @@
+# Copyright 2014: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from rally import objects
+from tests import test
+from tests.verification.verifiers import fakes
+
+
+class VerificationTestCase(test.TestCase):
+    def setUp(self):
+        super(VerificationTestCase, self).setUp()
+        self.db_obj = {
+            'id': 777,
+            'uuid': 'test_uuid',
+            'failures': 0, 'tests': 2, 'errors': 0, 'time': '0.54',
+            'details': {
+                'failures': 0, 'tests': 2, 'errors': 0, 'time': '0.54',
+                'test_cases': [
+                    {'classname': 'foo.Test',
+                     'name': 'foo_test[gate,negative]',
+                     'time': '0.25'},
+                    {'classname': 'bar.Test',
+                     'name': 'bar_test[gate,negative]',
+                     'time': '0.29'}]}}
+
+    @mock.patch('rally.objects.verification.db.verification_create')
+    def test_init_with_create(self, mock_create):
+        objects.Verification(deployment_uuid='some_deployment_uuid')
+        mock_create.assert_called_once_with('some_deployment_uuid')
+
+    @mock.patch('rally.objects.verification.db.verification_create')
+    def test_init_without_create(self, mock_create):
+        verification = objects.Verification(db_object=self.db_obj)
+
+        self.assertEqual(0, mock_create.call_count)
+        self.assertEqual(self.db_obj['failures'], verification.failures)
+        self.assertEqual(self.db_obj['tests'], verification.tests)
+        self.assertEqual(self.db_obj['errors'], verification.errors)
+        self.assertEqual(self.db_obj['time'], verification.time)
+
+    @mock.patch('rally.objects.verification.db.verification_get')
+    def test_get(self, mock_get):
+        objects.Verification.get(self.db_obj['id'])
+        mock_get.assert_called_once_with(self.db_obj['id'])
+
+    @mock.patch('rally.objects.verification.db.verification_delete')
+    @mock.patch('rally.objects.verification.db.verification_create')
+    def test_create_and_delete(self, mock_create, mock_delete):
+        verification = objects.Verification(db_object=self.db_obj)
+        verification.delete()
+        mock_delete.assert_called_once_with(self.db_obj['uuid'])
+
+    @mock.patch('rally.objects.verification.db.verification_update')
+    def test_set_failed(self, mock_update):
+        mock_update.return_value = self.db_obj
+        verification = objects.Verification(db_object=self.db_obj)
+        verification.set_failed()
+        mock_update.assert_called_once_with(self.db_obj['uuid'],
+                                            {'status': 'failed'})
+
+    @mock.patch('rally.objects.verification.db.verification_result_create')
+    @mock.patch('rally.objects.verification.db.verification_update')
+    def test_finish_verification(self, mock_update, mock_create):
+        verification = objects.Verification(db_object=self.db_obj)
+        fake_results = fakes.get_fake_test_case()
+        verification.finish_verification(
+            fake_results['total'],
+            fake_results['test_cases'])
+
+        expected_values = {'status': 'finished'}
+        expected_values.update(fake_results['total'])
+        mock_update.assert_called_with(self.db_obj['uuid'], expected_values)
+
+        expected_data = fake_results['total'].copy()
+        expected_data['test_cases'] = fake_results['test_cases']
+        mock_create.assert_called_once_with(verification.uuid, expected_data)

tests/orchestrator/test_api.py

@@ -212,9 +212,10 @@ class APITestCase(test.TestCase):
             mock.call(self.deploy_uuid, {'endpoints': self.endpoints})
         ])
 
+    @mock.patch('rally.orchestrator.api.objects.Verification')
     @mock.patch('rally.verification.verifiers.tempest.tempest.Tempest')
     @mock.patch('rally.objects.deploy.db.deployment_get')
-    def test_verify(self, mock_get, mock_tempest):
+    def test_verify(self, mock_get, mock_tempest, mock_verification):
         mock_tempest.return_value = self.tempest
         self.tempest.is_installed.return_value = True
         mock_get.return_value = self.deployment

tests/verification/verifiers/fake_log.xml

@@ -0,0 +1,13 @@
+<testsuite errors="0" failures="2" name="" tests="2" time="1.412">
+<testcase classname="fake.failed.TestCase" name="with_StringException[gate,negative]" time="0.706">
+<failure type="testtools.testresult.real._StringException">_StringException: Empty attachments:
+Oops...There was supposed to be fake traceback, but it is not.
+</failure>
+</testcase>
+<testcase classname="fake.successful.TestCase" name="fake_test[gate,negative]" time="0.706" />
+<testcase classname="" name="process-returncode" time="0.000">
+<failure type="testtools.testresult.real._StringException">_StringException: Binary content:
+traceback (test/plain; charset="utf8")
+</failure>
+</testcase>
+</testsuite>

tests/verification/verifiers/fakes.py

@@ -80,3 +80,31 @@ FAKE_CONFIG = {
         ('default_network', '10.0.0.0/24'),
         ('api_version', '2.0')]
 }
+
+
+def get_fake_test_case():
+    return {
+        'total': {
+            'failures': 1,
+            'tests': 2,
+            'errors': 0,
+            'time': 1.412},
+        'test_cases': {
+            'fake.failed.TestCase.with_StringException[gate,negative]': {
+                'name':
+                    'fake.failed.TestCase.with_StringException[gate,negative]',
+                'failure': {
+                    'type': 'testtools.testresult.real._StringException',
+                    'log':
+                        ('_StringException: Empty attachments:\nOops...There '
+                         'was supposed to be fake traceback, but it is not.\n')
+                },
+                'time': 0.706,
+                'status': 'FAIL'},
+            'fake.successful.TestCase.fake_test[gate,negative]': {
+                'name': 'fake.successful.TestCase.fake_test[gate,negative]',
+                'time': 0.706,
+                'status': 'OK'
+            }
+        }
+    }

tests/verification/verifiers/test_tempest.py

@@ -31,6 +31,9 @@ class TempestTestCase(test.TestCase):
     def setUp(self):
         super(TempestTestCase, self).setUp()
         self.verifier = tempest.Tempest('fake_deploy_id')
+
+        self.verifier.verification = mock.MagicMock()
+
         self.verifier.lock_path = 'fake_lock_path'
         self.conf_opts = (
             ('compute', [
@@ -86,28 +89,30 @@ class TempestTestCase(test.TestCase):
 
         result = self.verifier.is_installed()
 
-        mock_exists.assert_called_once_with(self.verifier.tempest_path)
+        mock_exists.assert_called_once_with(
+            os.path.join(self.verifier.tempest_path, '.venv'))
         self.assertTrue(result)
 
     @mock.patch('rally.verification.verifiers.tempest.tempest.subprocess')
     def test__clone(self, mock_sp):
         self.verifier._clone()
-        mock_sp.call.assert_called_once_with(
+        mock_sp.check_call.assert_called_once_with(
             ['git', 'clone', 'git://github.com/openstack/tempest',
              tempest.Tempest.tempest_base_path])
 
-    @mock.patch('rally.verification.verifiers.tempest.tempest.subprocess')
+    @mock.patch(TEMPEST_PATH + '.Tempest._install_venv')
+    @mock.patch(TEMPEST_PATH + '.subprocess')
     @mock.patch('os.path.exists')
     @mock.patch('shutil.copytree')
-    def test_install(self, mock_copytree, mock_exists, mock_sp):
-        mock_exists.side_effect = (True, False)
+    def test_install(self, mock_copytree, mock_exists, mock_sp, mock_venv):
+        mock_exists.side_effect = (False, True, False)
         # simulate tempest is clonned but is not installed for current deploy
 
         self.verifier.install()
         mock_copytree.assert_called_once_with(
             tempest.Tempest.tempest_base_path,
             self.verifier.tempest_path)
-        mock_sp.Popen.assert_called_once_with(
+        mock_sp.check_call.assert_called_once_with(
             'git checkout master; git remote update; git pull',
             cwd=os.path.join(self.verifier.tempest_path, 'tempest'),
             shell=True)
@@ -123,31 +128,42 @@ class TempestTestCase(test.TestCase):
     @mock.patch('shutil.rmtree')
     @mock.patch(TEMPEST_PATH + '.subprocess')
     def test__run(self, mock_sp, mock_rmtree, mock_env):
-        self.verifier._run('smoke', None)
-        fake_call = '%(venv)s testr run --parallel --subunit smoke  | ' \
-                    '%(venv)s subunit-2to1 ' \
-                    '| %(venv)s %(tempest_path)s/tools/colorizer.py' % {
-                        'venv': self.verifier._venv_wrapper,
-                        'tempest_path': self.verifier.tempest_path}
+        self.verifier._run('smoke')
+        fake_call = (
+            '%(venv)s testr run --parallel --subunit smoke '
+            '| %(venv)s subunit2junitxml --forward '
+            '--output-to=%(tempest_path)s/testr_log.xml '
+            '| %(venv)s subunit-2to1 '
+            '| %(venv)s %(tempest_path)s/tools/colorizer.py' % {
+                'venv': self.verifier._venv_wrapper,
+                'tempest_path': self.verifier.tempest_path})
         mock_sp.check_call.assert_called_once_with(
             fake_call, env=mock_env(), cwd=self.verifier.tempest_path,
             shell=True)
 
-    @mock.patch(TEMPEST_PATH + '.Tempest._check_venv_existence')
+    @mock.patch(TEMPEST_PATH + '.Tempest._save_results')
+    @mock.patch(TEMPEST_PATH + '.Tempest._prepare_and_run')
+    def test_verify(self, mock_run, mock_save_results):
+        self.verifier.verify(set_name=self.set_name, regex=None,
+                             options=self.conf_opts)
+
+        mock_run.assert_called_once_with('smoke', None, self.conf_opts)
+
     @mock.patch(TEMPEST_PATH + '.Tempest._check_testr_initialization')
     @mock.patch(TEMPEST_PATH + '.Tempest._run')
     @mock.patch(TEMPEST_PATH + '.Tempest._write_config')
     @mock.patch(TEMPEST_PATH + '.Tempest._generate_config')
-    def test_verify(self, mock_gen, mock_write, mock_run, mock_check_testr,
-                    mock_check_venv):
+    def test__prepare_and_run(self, mock_gen, mock_write, mock_run,
+                              mock_check_testr):
         mock_gen.return_value = 'fake_conf'
 
-        self.verifier.verify(set_name=self.set_name, regex=None,
-                             options=self.conf_opts)
+        self.verifier._prepare_and_run(set_name=self.set_name,
+                                       regex=None,
+                                       options=self.conf_opts)
 
         mock_gen.assert_called_once_with(self.conf_opts)
         mock_write.assert_called_once_with('fake_conf')
-        mock_run.assert_called_once_with('smoke', None)
+        mock_run.assert_called_once_with('smoke')
 
     @mock.patch('os.environ')
     def test__generate_env(self, mock_env):
@@ -162,26 +178,27 @@ class TempestTestCase(test.TestCase):
 
     @mock.patch('os.path.isdir')
     @mock.patch(TEMPEST_PATH + '.subprocess')
-    def test__check_venv_existence_when_venv_exists(self, mock_sp, mock_isdir):
+    def test__venv_install_when_venv_exists(self, mock_sp, mock_isdir):
         mock_isdir.return_value = True
-        self.verifier._check_venv_existence()
+        self.verifier._install_venv()
 
         mock_isdir.assert_called_once_with(
             os.path.join(self.verifier.tempest_path, '.venv'))
         self.assertEqual(0, mock_sp.call_count)
 
     @mock.patch('os.path.isdir')
-    @mock.patch(TEMPEST_PATH + '.subprocess.call')
-    def test__check_venv_existence_when_venv_not_exist(self, mock_sp,
-                                                       mock_isdir):
+    @mock.patch(TEMPEST_PATH + '.subprocess')
+    def test__venv_install_when_venv_not_exist(self, mock_sp, mock_isdir):
         mock_isdir.return_value = False
-        self.verifier._check_venv_existence()
+        self.verifier._install_venv()
 
         mock_isdir.assert_called_once_with(
             os.path.join(self.verifier.tempest_path, '.venv'))
-        mock_sp.assert_called_once_with('python ./tools/install_venv.py',
-                                        shell=True,
-                                        cwd=self.verifier.tempest_path)
+        mock_sp.check_call.assert_has_calls([
+            mock.call('python ./tools/install_venv.py', shell=True,
+                      cwd=self.verifier.tempest_path),
+            mock.call('%s pip install junitxml' % self.verifier._venv_wrapper,
+                      shell=True, cwd=self.verifier.tempest_path)])
 
     @mock.patch('os.path.isdir')
     @mock.patch(TEMPEST_PATH + '.subprocess')
@@ -206,3 +223,26 @@ class TempestTestCase(test.TestCase):
         mock_sp.assert_called_once_with(
             '%s testr init' % self.verifier._venv_wrapper, shell=True,
             cwd=self.verifier.tempest_path)
+
+    @mock.patch('xml.dom.minidom')
+    @mock.patch('os.path.isfile')
+    def test__save_results_without_log_file(self, mock_isfile, mock_minidom):
+        mock_isfile.return_value = False
+
+        self.verifier._save_results()
+
+        mock_isfile.assert_called_once_with(self.verifier.log_file)
+        self.assertEqual(0, mock_minidom.call_count)
+
+    @mock.patch('os.path.isfile')
+    def test__save_results_with_log_file(self, mock_isfile):
+        mock_isfile.return_value = True
+        self.verifier.log_file = os.path.join(os.path.dirname(__file__),
+                                              'fake_log.xml')
+
+        self.verifier._save_results()
+        mock_isfile.assert_called_once_with(self.verifier.log_file)
+        fake_test_case = fakes.get_fake_test_case()
+        self.verifier.verification.finish_verification.assert_called_once_with(
+            total=fake_test_case['total'],
+            test_cases=fake_test_case['test_cases'])