diff --git a/rally/cmd/api.py b/rally/cmd/api.py
index 8b3482c63c..ce3649b8c3 100644
--- a/rally/cmd/api.py
+++ b/rally/cmd/api.py
@@ -32,7 +32,7 @@ LOG = log.getLogger(__name__)
def main():
- # Initialize configuation and logging.
+ # Initialize configuration and logging.
CONF(sys.argv[1:], project='rally')
log.setup('rally')
# Prepare application and bind to the service socket.
diff --git a/rally/cmd/commands/verify.py b/rally/cmd/commands/verify.py
index 38b8e0e9a7..036dfc89ff 100644
--- a/rally/cmd/commands/verify.py
+++ b/rally/cmd/commands/verify.py
@@ -15,6 +15,7 @@
""" Rally command: verify """
+import csv
import json
import os
@@ -30,6 +31,7 @@ from rally.i18n import _
from rally import objects
from rally.openstack.common import cliutils as common_cliutils
from rally.orchestrator import api
+from rally.verification.verifiers.tempest import diff
from rally.verification.verifiers.tempest import json2html
@@ -71,30 +73,30 @@ class VerifyCommands(object):
return (1)
verification = api.verify(deploy_id, set_name, regex, tempest_config)
if do_use:
- use.UseCommands().verification(verification['uuid'])
+ use.UseCommands().verification(verification["uuid"])
def list(self):
"""Display all verifications table, started and finished."""
- fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', 'Failures',
- 'Created at', 'Status']
+ fields = ["UUID", "Deployment UUID", "Set name", "Tests", "Failures",
+ "Created at", "Status"]
verifications = db.verification_list()
if verifications:
common_cliutils.print_list(verifications, fields,
- sortby_index=fields.index('Created at'))
+ sortby_index=fields.index("Created at"))
else:
print(_("There are no results from verifier. To run a verifier, "
"use:\nrally verify start"))
- @cliutils.args('--uuid', type=str, dest='verification_uuid',
- help='UUID of the verification')
- @cliutils.args('--html', action='store_true', dest='output_html',
- help=('Results will be in html format'))
- @cliutils.args('--json', action='store_true', dest='output_json',
- help=('Results will be in json format'))
- @cliutils.args('--output-file', type=str, required=False,
- dest='output_file',
- help='If specified, output will be saved to given file')
+ @cliutils.args("--uuid", type=str, dest="verification_uuid",
+ help="UUID of the verification")
+ @cliutils.args("--html", action="store_true", dest="output_html",
+ help=("Results will be in html format"))
+ @cliutils.args("--json", action="store_true", dest="output_json",
+ help=("Results will be in json format"))
+ @cliutils.args("--output-file", type=str, required=False,
+ dest="output_file",
+ help="If specified, output will be saved to given file")
@envutils.with_default_verification_id
def results(self, verification_uuid=None, output_file=None,
output_html=None, output_json=None):
@@ -107,13 +109,13 @@ class VerifyCommands(object):
"""
try:
- results = db.verification_result_get(verification_uuid)['data']
+ results = db.verification_result_get(verification_uuid)["data"]
except exceptions.NotFoundException as e:
print(six.text_type(e))
return 1
- result = ''
- if len(filter(lambda x: bool(x), [output_json, output_html])) > 1:
+ result = ""
+ if output_json + output_html > 1:
print("Please specify only one output format.")
elif output_html:
result = json2html.main(results)
@@ -122,24 +124,24 @@ class VerifyCommands(object):
if output_file:
output_file = os.path.expanduser(output_file)
- with open(output_file, 'wb') as f:
+ with open(output_file, "wb") as f:
f.write(result)
else:
print(result)
- @cliutils.args('--uuid', dest='verification_uuid', type=str,
+ @cliutils.args("--uuid", dest="verification_uuid", type=str,
required=False,
- help='UUID of a verification')
- @cliutils.args('--sort-by', dest='sort_by', type=str, required=False,
- help='Tests can be sorted by "name" or "duration"')
- @cliutils.args('--detailed', dest='detailed', action='store_true',
- required=False, help='Prints traceback of failed tests')
+ help="UUID of a verification")
+ @cliutils.args("--sort-by", dest="sort_by", type=str, required=False,
+ help="Tests can be sorted by 'name' or 'duration'")
+ @cliutils.args("--detailed", dest="detailed", action="store_true",
+ required=False, help="Prints traceback of failed tests")
@envutils.with_default_verification_id
- def show(self, verification_uuid=None, sort_by='name', detailed=False):
+ def show(self, verification_uuid=None, sort_by="name", detailed=False):
"""Display results table of the verification."""
try:
- sortby_index = ('name', 'duration').index(sort_by)
+ sortby_index = ("name", "duration").index(sort_by)
except ValueError:
print("Sorry, but verification results can't be sorted "
"by '%s'." % sort_by)
@@ -153,42 +155,104 @@ class VerifyCommands(object):
return 1
print ("Total results of verification:\n")
- total_fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests',
- 'Failures', 'Created at', 'Status']
+ total_fields = ["UUID", "Deployment UUID", "Set name", "Tests",
+ "Failures", "Created at", "Status"]
common_cliutils.print_list([verification], fields=total_fields)
print ("\nTests:\n")
- fields = ['name', 'time', 'status']
+ fields = ["name", "time", "status"]
values = map(objects.Verification,
- six.itervalues(tests.data['test_cases']))
+ six.itervalues(tests.data["test_cases"]))
common_cliutils.print_list(values, fields, sortby_index=sortby_index)
if detailed:
- for test in six.itervalues(tests.data['test_cases']):
- if test['status'] == 'FAIL':
+ for test in six.itervalues(tests.data["test_cases"]):
+ if test["status"] == "FAIL":
formatted_test = (
- '====================================================='
- '=================\n'
- 'FAIL: %(name)s\n'
- 'Time: %(time)s\n'
- 'Type: %(type)s\n'
- '-----------------------------------------------------'
- '-----------------\n'
- '%(log)s\n'
+ "====================================================="
+ "=================\n"
+ "FAIL: %(name)s\n"
+ "Time: %(time)s\n"
+ "Type: %(type)s\n"
+ "-----------------------------------------------------"
+ "-----------------\n"
+ "%(log)s\n"
) % {
- 'name': test['name'], 'time': test['time'],
- 'type': test['failure']['type'],
- 'log': test['failure']['log']}
+ "name": test["name"], "time": test["time"],
+ "type": test["failure"]["type"],
+ "log": test["failure"]["log"]}
print (formatted_test)
- @cliutils.args('--uuid', dest='verification_uuid', type=str,
+ @cliutils.args("--uuid", dest="verification_uuid", type=str,
required=False,
- help='UUID of a verification')
- @cliutils.args('--sort-by', dest='sort_by', type=str, required=False,
- help='Tests can be sorted by "name" or "duration"')
+ help="UUID of a verification")
+ @cliutils.args("--sort-by", dest="sort_by", type=str, required=False,
+ help="Tests can be sorted by 'name' or 'duration'")
@envutils.with_default_verification_id
- def detailed(self, verification_uuid=None, sort_by='name'):
+ def detailed(self, verification_uuid=None, sort_by="name"):
"""Display results table of verification with detailed errors."""
self.show(verification_uuid, sort_by, True)
+
+ @cliutils.args("--uuid-1", type=str, dest="uuid1",
+ help="UUID of the first verification")
+ @cliutils.args("--uuid-2", type=str, dest="uuid2",
+ help="UUID of the second verification")
+ @cliutils.args("--csv", action="store_true", dest="output_csv",
+ help=("Save results in csv format to specified file"))
+ @cliutils.args("--html", action="store_true", dest="output_html",
+ help=("Save results in html format to specified file"))
+ @cliutils.args("--json", action="store_true", dest="output_json",
+ help=("Save results in json format to specified file"))
+ @cliutils.args("--output-file", type=str, required=False,
+ dest="output_file",
+ help="If specified, output will be saved to given file")
+ @cliutils.args("--threshold", type=int, required=False,
+ dest="threshold", default=0,
+ help="If specified, timing differences must exceed this "
+ "percentage threshold to be included in output")
+ def compare(self, uuid1=None, uuid2=None,
+ output_file=None, output_csv=None, output_html=None,
+ output_json=None, threshold=0):
+ """Compare two verification results.
+
+ :param uuid1: First Verification UUID
+ :param uuid2: Second Verification UUID
+ :param output_file: If specified, output will be saved to given file
+ :param output_csv: Save results in csv format to the specified file
+ :param output_html: Save results in html format to the specified file
+ :param output_json: Save results in json format to the specified file
+ (Default)
+ :param threshold: Timing difference threshold percentage
+ """
+
+ try:
+ results1 = db.verification_result_get(uuid1)["data"]["test_cases"]
+ results2 = db.verification_result_get(uuid2)["data"]["test_cases"]
+ _diff = diff.Diff(results1, results2, threshold)
+ except exceptions.NotFoundException as e:
+ print(six.text_type(e))
+ return 1
+
+ result = ""
+ if output_json + output_html + output_csv > 1:
+ print("Please specify only one output format, either --json, "
+ "--html or --csv.")
+ return 1
+ elif output_html:
+ result = _diff.to_html()
+ elif output_csv:
+ result = _diff.to_csv()
+ else:
+ result = _diff.to_json()
+
+ if output_file:
+ with open(output_file, "wb") as f:
+ if output_csv:
+ writer = csv.writer(f, dialect="excel")
+ writer.writerows(result)
+ else:
+ f.write(result)
+ else:
+ print(result)
diff --git a/rally/verification/verifiers/tempest/compare2html.py b/rally/verification/verifiers/tempest/compare2html.py
new file mode 100644
index 0000000000..1cd767691d
--- /dev/null
+++ b/rally/verification/verifiers/tempest/compare2html.py
@@ -0,0 +1,41 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Output verification comparison results in html."""
+
+import os
+
+import mako.template
+
+__description__ = "List differences between two verification runs"
+__title__ = "Verification Comparison"
+__version__ = "0.1"
+
+
+def create_report(results):
+ template_kw = {
+ "heading": {
+ "title": __title__,
+ "description": __description__,
+ "parameters": [("Difference Count", len(results))]
+ },
+ "generator": "compare2html %s" % __version__,
+ "results": results
+ }
+
+ template_path = os.path.join(os.path.dirname(__file__),
+ "report_templates",
+ "compare.mako")
+
+ with open(template_path) as f:
+ template = mako.template.Template(f.read(), strict_undefined=True)
+ output = template.render(**template_kw)
+ return output.encode('utf8')
diff --git a/rally/verification/verifiers/tempest/diff.py b/rally/verification/verifiers/tempest/diff.py
new file mode 100644
index 0000000000..993a3cb671
--- /dev/null
+++ b/rally/verification/verifiers/tempest/diff.py
@@ -0,0 +1,108 @@
+# Copyright 2014 Dell Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+
+import compare2html
+
+
+class Diff(object):
+
+ def __init__(self, test_cases1, test_cases2, threshold):
+ """Compare two verification results.
+
+ Compares two verification results and emits
+ desired output, csv, html, json or pprint.
+
+ :param test_cases1: older verification json
+ :param test_cases2: newer verification json
+ :param threshold: test time difference percentage threshold
+
+ """
+ self.threshold = threshold
+ self.diffs = self._compare(test_cases1, test_cases2)
+
+ def _compare(self, tc1, tc2):
+ """Compare two verification results.
+
+ :param tc1: first verification test cases json
+ :param tc2: second verification test cases json
+
+ Typical test case json schema:
+ "test_case_key": {
+ "failure": {
+ "log": ""
+ },
+ "name": "",
+ "output": "",
+ "status": "",
+ "time": 0.0
+ }
+ """
+ names1 = sorted(tc1)
+ names2 = sorted(tc2)
+
+ diffs = []
+ i = j = 0
+ while i < len(names1) and j < len(names2):
+ name1 = names1[i] if i < len(names1) else None
+ name2 = names2[j] if j < len(names2) else None
+ if name1 and name2 and name1 == name2:
+ diffs.extend(self._diff_values(name1, tc1[name1], tc2[name2]))
+ i += 1
+ j += 1
+
+ elif (not name1) or (name1 > name2):
+ diffs.append({"type": "new_test", "test_name": name2})
+ j += 1
+ else:
+ diffs.append({"type": "removed_test", "test_name": name1})
+ i += 1
+
+ return diffs
+
+ def _diff_values(self, name, result1, result2):
+ th = self.threshold
+ fields = ["status", "time", "output"]
+ diffs = []
+ for field in fields:
+ val1 = result1[field]
+ val2 = result2[field]
+ if val1 != val2 and not (field == "time"
+ and abs(((val2 - val1) / val1) * 100)
+ < th):
+ diffs.append({
+ "field": field,
+ "type": "value_changed",
+ "test_name": name,
+ "val1": val1,
+ "val2": val2
+ })
+ return diffs
+
+ def to_csv(self):
+ rows = (("Type", "Field", "Value 1", "Value 2", "Test Name"),)
+ for res in self.diffs:
+ row = (res.get("type"), res.get("field", ""),
+ res.get("val1", ""), res.get("val2", ""),
+ res.get("test_name"))
+ rows = rows + (row,)
+ return rows
+
+ def to_json(self):
+ return json.dumps(self.diffs, sort_keys=True, indent=4)
+
+ def to_html(self):
+ return compare2html.create_report(self.diffs)
diff --git a/rally/verification/verifiers/tempest/report_templates/compare.mako b/rally/verification/verifiers/tempest/report_templates/compare.mako
new file mode 100644
index 0000000000..477cc3d4d5
--- /dev/null
+++ b/rally/verification/verifiers/tempest/report_templates/compare.mako
@@ -0,0 +1,165 @@
+
+
+
+
+ ${heading["title"]}
+
+
+
+
+
+
+
+
+
${heading["title"]}
+ % for name, value in heading["parameters"]:
+
${name}: ${value}
+ % endfor
+
${heading["description"]}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ % for diff in results:
+
+ ${diff.get("type")} |
+ ${diff.get("field", "")} |
+ ${diff.get("val1", "")} |
+ ${diff.get("val2", "")} |
+ ${diff.get("test_name")} |
+
+ % endfor
+
+
+
+
diff --git a/tests/unit/cmd/commands/test_verify.py b/tests/unit/cmd/commands/test_verify.py
index 502e479a83..cb68c63872 100644
--- a/tests/unit/cmd/commands/test_verify.py
+++ b/tests/unit/cmd/commands/test_verify.py
@@ -31,41 +31,41 @@ class VerifyCommandsTestCase(test.TestCase):
self.verify = verify.VerifyCommands()
self.image1 = mock.Mock()
- self.image1.name = 'cirros-1'
- self.image1.id = 'fake_image_id_1'
+ self.image1.name = "cirros-1"
+ self.image1.id = "fake_image_id_1"
self.image2 = mock.Mock()
- self.image2.id = 'fake_image_id_2'
- self.image2.name = 'cirros-2'
+ self.image2.id = "fake_image_id_2"
+ self.image2.name = "cirros-2"
self.flavor1 = mock.Mock()
self.flavor2 = mock.Mock()
- self.flavor1.id = 'fake_flavor_id_1'
- self.flavor2.id = 'fake_flavor_id_2'
+ self.flavor1.id = "fake_flavor_id_1"
+ self.flavor2.id = "fake_flavor_id_2"
self.flavor1.ram = 128
self.flavor2.ram = 64
- @mock.patch('rally.osclients.Clients')
- @mock.patch('rally.orchestrator.api.verify')
+ @mock.patch("rally.osclients.Clients")
+ @mock.patch("rally.orchestrator.api.verify")
def test_start(self, mock_verify, mock_clients):
- deploy_id = '0fba91c6-82d5-4ce1-bd00-5d7c989552d9'
+ deploy_id = "0fba91c6-82d5-4ce1-bd00-5d7c989552d9"
mock_clients().glance().images.list.return_value = [
self.image1, self.image2]
mock_clients().nova().flavors.list.return_value = [
self.flavor1, self.flavor2]
self.verify.start(deploy_id=deploy_id)
- default_set_name = 'smoke'
+ default_set_name = "smoke"
default_regex = None
mock_verify.assert_called_once_with(deploy_id,
default_set_name, default_regex,
None)
- @mock.patch('rally.osclients.Clients')
- @mock.patch('rally.orchestrator.api.verify')
+ @mock.patch("rally.osclients.Clients")
+ @mock.patch("rally.orchestrator.api.verify")
def test_start_with_user_specified_tempest_config(self, mock_verify,
mock_clients):
- deploy_id = '0fba91c6-82d5-4ce1-bd00-5d7c989552d9'
+ deploy_id = "0fba91c6-82d5-4ce1-bd00-5d7c989552d9"
mock_clients().glance().images.list.return_value = [
self.image1, self.image2]
mock_clients().nova().flavors.list.return_value = [
@@ -73,7 +73,7 @@ class VerifyCommandsTestCase(test.TestCase):
tempest_config = tempfile.NamedTemporaryFile()
self.verify.start(deploy_id=deploy_id,
tempest_config=tempest_config.name)
- default_set_name = 'smoke'
+ default_set_name = "smoke"
default_regex = None
mock_verify.assert_called_once_with(deploy_id,
@@ -81,114 +81,212 @@ class VerifyCommandsTestCase(test.TestCase):
tempest_config.name)
tempest_config.close()
- @mock.patch('rally.orchestrator.api.verify')
+ @mock.patch("rally.orchestrator.api.verify")
def test_start_with_wrong_set_name(self, mock_verify):
- deploy_id = 'f2009aae-6ef3-468e-96b2-3c987d584010'
+ deploy_id = "f2009aae-6ef3-468e-96b2-3c987d584010"
- wrong_set_name = 'unexpected_value'
+ wrong_set_name = "unexpected_value"
self.verify.start(deploy_id, wrong_set_name)
self.assertNotIn(wrong_set_name, consts.TEMPEST_TEST_SETS)
self.assertFalse(mock_verify.called)
- @mock.patch('rally.openstack.common.cliutils.print_list')
- @mock.patch('rally.db.verification_list')
+ @mock.patch("rally.openstack.common.cliutils.print_list")
+ @mock.patch("rally.db.verification_list")
def test_list(self, mock_db_verification_list, mock_print_list):
- fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', 'Failures',
- 'Created at', 'Status']
- verifications = {'dummy': []}
+ fields = ["UUID", "Deployment UUID", "Set name", "Tests", "Failures",
+ "Created at", "Status"]
+ verifications = {"dummy": []}
mock_db_verification_list.return_value = verifications
self.verify.list()
mock_db_verification_list.assert_called_once_with()
mock_print_list.assert_called_once_with(verifications, fields,
sortby_index=fields.index(
- 'Created at'))
+ "Created at"))
- @mock.patch('rally.openstack.common.cliutils.print_list')
- @mock.patch('rally.db.verification_get')
- @mock.patch('rally.db.verification_result_get')
- @mock.patch('rally.objects.Verification')
+ @mock.patch("rally.openstack.common.cliutils.print_list")
+ @mock.patch("rally.db.verification_get")
+ @mock.patch("rally.db.verification_result_get")
+ @mock.patch("rally.objects.Verification")
def test_show(self, mock_obj_verification,
mock_verification_result_get, mock_verification_get,
mock_print_list):
class Test_dummy():
- data = {'test_cases': {'test_a': {'name': 'test_a', 'time': 20,
- 'status': 'PASS'},
- 'test_b': {'name': 'test_b', 'time': 20,
- 'status': 'SKIP'},
- 'test_c': {'name': 'test_c', 'time': 20,
- 'status': 'FAIL'}}}
+ data = {"test_cases": {"test_a": {"name": "test_a", "time": 20,
+ "status": "PASS"},
+ "test_b": {"name": "test_b", "time": 20,
+ "status": "SKIP"},
+ "test_c": {"name": "test_c", "time": 20,
+ "status": "FAIL"}}}
- verification_id = '39121186-b9a4-421d-b094-6c6b270cf9e9'
- total_fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests',
- 'Failures', 'Created at', 'Status']
- fields = ['name', 'time', 'status']
+ verification_id = "39121186-b9a4-421d-b094-6c6b270cf9e9"
+ total_fields = ["UUID", "Deployment UUID", "Set name", "Tests",
+ "Failures", "Created at", "Status"]
+ fields = ["name", "time", "status"]
verification = mock.MagicMock()
tests = Test_dummy()
mock_verification_result_get.return_value = tests
mock_verification_get.return_value = verification
mock_obj_verification.return_value = 1
values = map(objects.Verification,
- six.itervalues(tests.data['test_cases']))
+ six.itervalues(tests.data["test_cases"]))
self.verify.show(verification_id)
- mock_print_list.assert_any_call(
- [verification], fields=total_fields)
+ mock_print_list.assert_any_call([verification], fields=total_fields)
mock_verification_get.assert_called_once_with(verification_id)
mock_verification_result_get.assert_called_once_with(verification_id)
mock_print_list.assert_any_call(values, fields, sortby_index=0)
- @mock.patch('rally.db.verification_result_get', return_value={'data': {}})
- @mock.patch('json.dumps')
+ @mock.patch("rally.db.verification_result_get", return_value={"data": {}})
+ @mock.patch("json.dumps")
def test_results(self, mock_json_dumps, mock_db_result_get):
- verification_uuid = 'a0231bdf-6a4e-4daf-8ab1-ae076f75f070'
- self.verify.results(verification_uuid, output_json=True)
+ verification_uuid = "a0231bdf-6a4e-4daf-8ab1-ae076f75f070"
+ self.verify.results(verification_uuid, output_html=False,
+ output_json=True)
mock_db_result_get.assert_called_once_with(verification_uuid)
mock_json_dumps.assert_called_once_with({}, sort_keys=True, indent=4)
- @mock.patch('rally.db.verification_result_get')
+ @mock.patch("rally.db.verification_result_get")
def test_results_verification_not_found(self, mock_db_result_get):
- verification_uuid = '9044ced5-9c84-4666-8a8f-4b73a2b62acb'
+ verification_uuid = "9044ced5-9c84-4666-8a8f-4b73a2b62acb"
mock_db_result_get.side_effect = exceptions.NotFoundException()
- self.assertEqual(self.verify.results(verification_uuid), 1)
+ self.assertEqual(self.verify.results(verification_uuid,
+ output_html=False,
+ output_json=True), 1)
mock_db_result_get.assert_called_once_with(verification_uuid)
- @mock.patch('rally.cmd.commands.verify.open', create=True)
- @mock.patch('rally.db.verification_result_get', return_value={'data': {}})
+ @mock.patch("rally.cmd.commands.verify.open", create=True)
+ @mock.patch("rally.db.verification_result_get", return_value={"data": {}})
def test_results_with_output_json_and_output_file(self,
mock_db_result_get,
mock_open):
mock_open.return_value = mock.MagicMock()
- verification_uuid = '94615cd4-ff45-4123-86bd-4b0741541d09'
- self.verify.results(verification_uuid, output_file='results',
- output_json=True)
+ verification_uuid = "94615cd4-ff45-4123-86bd-4b0741541d09"
+ self.verify.results(verification_uuid, output_file="results",
+ output_html=False, output_json=True)
mock_db_result_get.assert_called_once_with(verification_uuid)
- mock_open.assert_called_once_with('results', 'wb')
+ mock_open.assert_called_once_with("results", "wb")
fake_file = mock_open.return_value.__enter__.return_value
- fake_file.write.assert_called_once_with('{}')
+ fake_file.write.assert_called_once_with("{}")
- @mock.patch('rally.cmd.commands.verify.open', create=True)
- @mock.patch('rally.db.verification_result_get')
- @mock.patch('rally.verification.verifiers.tempest.json2html.main',
- return_value='')
+ @mock.patch("rally.cmd.commands.verify.open", create=True)
+ @mock.patch("rally.db.verification_result_get")
+ @mock.patch("rally.verification.verifiers.tempest.json2html.main",
+ return_value="")
def test_results_with_output_html_and_output_file(self,
mock_json2html_main,
mock_db_result_get,
mock_open):
mock_open.return_value = mock.MagicMock()
- verification_uuid = '7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa'
+ verification_uuid = "7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa"
fake_data = {}
- results = {'data': fake_data}
+ results = {"data": fake_data}
mock_db_result_get.return_value = results
self.verify.results(verification_uuid, output_html=True,
- output_file='results')
+ output_json=False, output_file="results")
mock_db_result_get.assert_called_once_with(verification_uuid)
mock_json2html_main.assert_called_once_with(fake_data)
- mock_open.assert_called_once_with('results', 'wb')
+ mock_open.assert_called_once_with("results", "wb")
fake_file = mock_open.return_value.__enter__.return_value
- fake_file.write.assert_called_once_with('')
+ fake_file.write.assert_called_once_with("")
+
+ @mock.patch("rally.db.verification_result_get",
+ return_value={"data": {"test_cases": {}}})
+ @mock.patch("json.dumps")
+ def test_compare(self, mock_json_dumps, mock_db_result_get):
+ uuid1 = "8eda1b10-c8a4-4316-9603-8468ff1d1560"
+ uuid2 = "f6ef0a98-1b18-452f-a6a7-922555c2e326"
+ self.verify.compare(uuid1, uuid2, output_csv=False, output_html=False,
+ output_json=True)
+
+ fake_data = []
+ calls = [mock.call(uuid1),
+ mock.call(uuid2)]
+ mock_db_result_get.assert_has_calls(calls, True)
+ mock_json_dumps.assert_called_once_with(fake_data, sort_keys=True,
+ indent=4)
+
+ @mock.patch("rally.db.verification_result_get",
+ side_effect=exceptions.NotFoundException())
+ def test_compare_verification_not_found(self, mock_db_result_get):
+ uuid1 = "f7dc82da-31a6-4d40-bbf8-6d366d58960f"
+ uuid2 = "2f8a05f3-d310-4f02-aabf-e1165aaa5f9c"
+
+ self.assertEqual(self.verify.compare(uuid1, uuid2, output_csv=False,
+ output_html=False,
+ output_json=True), 1)
+
+ mock_db_result_get.assert_called_once_with(uuid1)
+
+ @mock.patch("rally.cmd.commands.verify.open", create=True)
+ @mock.patch("rally.db.verification_result_get",
+ return_value={"data": {"test_cases": {}}})
+ def test_compare_with_output_csv_and_output_file(self,
+ mock_db_result_get,
+ mock_open):
+
+ fake_string = "Type,Field,Value 1,Value 2,Test Name\r\n"
+ uuid1 = "5e744557-4c3a-414f-9afb-7d3d8708028f"
+ uuid2 = "efe1c74d-a632-476e-bb6a-55a9aa9cf76b"
+ self.verify.compare(uuid1, uuid2, output_file="results",
+ output_csv=True, output_html=False,
+ output_json=False)
+
+ calls = [mock.call(uuid1),
+ mock.call(uuid2)]
+ mock_db_result_get.assert_has_calls(calls, True)
+ mock_open.assert_called_once_with("results", "wb")
+ fake_file = mock_open.return_value.__enter__.return_value
+ fake_file.write.assert_called_once_with(fake_string)
+
+ @mock.patch("rally.cmd.commands.verify.open", create=True)
+ @mock.patch("rally.db.verification_result_get",
+ return_value={"data": {"test_cases": {}}})
+ def test_compare_with_output_json_and_output_file(self,
+ mock_db_result_get,
+ mock_open):
+ fake_json_string = "[]"
+ uuid1 = "0505e33a-738d-4474-a611-9db21547d863"
+ uuid2 = "b1908417-934e-481c-8d23-bc0badad39ed"
+ self.verify.compare(uuid1, uuid2, output_file="results",
+ output_csv=False, output_html=False,
+ output_json=True)
+
+ calls = [mock.call(uuid1),
+ mock.call(uuid2)]
+ mock_db_result_get.assert_has_calls(calls, True)
+ mock_open.assert_called_once_with("results", "wb")
+ fake_file = mock_open.return_value.__enter__.return_value
+ fake_file.write.assert_called_once_with(fake_json_string)
+
+ @mock.patch("rally.cmd.commands.verify.open", create=True)
+ @mock.patch("rally.db.verification_result_get")
+ @mock.patch(("rally.verification.verifiers.tempest."
+ "compare2html.create_report"), return_value="")
+ def test_compare_with_output_html_and_output_file(self,
+ mock_compare2html_create,
+ mock_db_result_get,
+ mock_open):
+
+ uuid1 = "cdf64228-77e9-414d-9d4b-f65e9d62c61f"
+ uuid2 = "39393eec-1b45-4103-8ec1-631edac4b8f0"
+ results = {"data": {"test_cases": {}}}
+ fake_data = []
+ self.verify.compare(uuid1, uuid2,
+ output_file="results",
+ output_csv=False, output_html=True,
+ output_json=False)
+ mock_db_result_get.return_value = results
+ calls = [mock.call(uuid1),
+ mock.call(uuid2)]
+ mock_db_result_get.assert_has_calls(calls, True)
+ mock_compare2html_create.assert_called_once_with(fake_data)
+
+ mock_open.assert_called_once_with("results", "wb")
+ fake_file = mock_open.return_value.__enter__.return_value
+ fake_file.write.assert_called_once_with("")
diff --git a/tests/unit/verification/verifiers/test_compare2html.py b/tests/unit/verification/verifiers/test_compare2html.py
new file mode 100644
index 0000000000..c2d8754505
--- /dev/null
+++ b/tests/unit/verification/verifiers/test_compare2html.py
@@ -0,0 +1,39 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+from rally.verification.verifiers.tempest import compare2html
+from tests.unit import test
+
+
+class Compare2HtmlTestCase(test.TestCase):
+
+ def test_main(self):
+ results = [{'val2': 0.0111, 'field': u'time', 'val1': 0.0222,
+ 'type': 'CHANGED', 'test_name': u'test.one'},
+ {'val2': 0.111, 'field': u'time', 'val1': 0.222,
+ 'type': 'CHANGED', 'test_name': u'test.two'},
+ {'val2': 1.11, 'field': u'time', 'val1': 2.22,
+ 'type': 'CHANGED', 'test_name': u'test.three'}]
+
+ fake_kw = {"heading":
+ {"title": compare2html.__title__,
+ "description": compare2html.__description__,
+ "parameters": [("Difference Count", len(results))]
+ },
+ "generator": "compare2html %s" % compare2html.__version__,
+ "results": results}
+
+ with mock.patch('mako.template.Template') as mock_mako:
+ compare2html.create_report(results)
+ mock_mako().render.assert_called_once_with(**fake_kw)
diff --git a/tests/unit/verification/verifiers/test_diff.py b/tests/unit/verification/verifiers/test_diff.py
new file mode 100644
index 0000000000..d623562b5d
--- /dev/null
+++ b/tests/unit/verification/verifiers/test_diff.py
@@ -0,0 +1,76 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from rally.verification.verifiers.tempest import diff
+from tests.unit import test
+
+
+class DiffTestCase(test.TestCase):
+
+ def test_main(self):
+ results1 = {'test.NONE': {'name': 'test.NONE',
+ 'output': 'test.NONE',
+ 'status': 'SKIPPED',
+ 'time': 0.000},
+ 'test.one': {'name': 'test.one',
+ 'output': 'test.one',
+ 'status': 'OK',
+ 'time': 0.111},
+ 'test.two': {'name': 'test.two',
+ 'output': 'test.two',
+ 'status': 'OK',
+ 'time': 0.222},
+ 'test.three': {'name': 'test.three',
+ 'output': 'test.three',
+ 'status': 'FAILED',
+ 'time': 0.333},
+ 'test.four': {'name': 'test.four',
+ 'output': 'test.four',
+ 'status': 'OK',
+ 'time': 0.444},
+ 'test.five': {'name': 'test.five',
+ 'output': 'test.five',
+ 'status': 'OK',
+ 'time': 0.555}
+ }
+
+ results2 = {'test.one': {'name': 'test.one',
+ 'output': 'test.one',
+ 'status': 'FAIL',
+ 'time': 0.1111},
+ 'test.two': {'name': 'test.two',
+ 'output': 'test.two',
+ 'status': 'OK',
+ 'time': 0.222},
+ 'test.three': {'name': 'test.three',
+ 'output': 'test.three',
+ 'status': 'OK',
+ 'time': 0.3333},
+ 'test.four': {'name': 'test.four',
+ 'output': 'test.four',
+ 'status': 'FAIL',
+ 'time': 0.4444},
+ 'test.five': {'name': 'test.five',
+ 'output': 'test.five',
+ 'status': 'OK',
+ 'time': 0.555},
+ 'test.six': {'name': 'test.six',
+ 'output': 'test.six',
+ 'status': 'OK',
+ 'time': 0.666}
+ }
+
+ diff_ = diff.Diff(results1, results2, 0)
+ assert len(diff_.diffs) == 8
+ assert diff_.to_csv() != ''
+ assert diff_.to_html() != ''
+ assert diff_.to_json() != ''
diff --git a/tools/rally.bash_completion b/tools/rally.bash_completion
index c050ab5eb3..9bce6982bf 100644
--- a/tools/rally.bash_completion
+++ b/tools/rally.bash_completion
@@ -31,6 +31,7 @@ _rally()
OPTS["show_keypairs"]="--deploy-id"
OPTS["show_networks"]="--deploy-id"
OPTS["show_secgroups"]="--deploy-id"
+ OPTS["verify_compare"]="--uuid-1 --uuid-2 --csv --html --json --output-file --threshold"
OPTS["verify_detailed"]="--uuid --sort-by"
OPTS["verify_list"]=""
OPTS["verify_results"]="--uuid --html --json --output-file"
@@ -74,4 +75,4 @@ _rally()
fi
return 0
}
-complete -F _rally rally
\ No newline at end of file
+complete -F _rally rally