From 5910775a6cb428727112fddaa708d35f6875cccd Mon Sep 17 00:00:00 2001 From: David Paterson Date: Fri, 24 Oct 2014 23:09:17 -0400 Subject: [PATCH] Add ability to compare two verification results Currently there is no built-in way in Rally to see the differences between two verification runs. This feature adds a compare sub-command to rally verify that takes two verification IDs as input and output the differences between two verifications in the desired format, html, csv, json or pprint. Output may be printed to std out or directed to file destination specified. Tasks - Add compare sub-command to rally verify. - Add ability to specify output format and file name of comparison report. Initial support will allow for output to either HTML, pretty print or CSV. - Create rally/verification/verifiers/tempest/diff.py that handles all the comparing and output of results. - Create required mako template (compare.mako) and renderer (compare2html.py) class for HTML output. - Added percentage threshold argument for determining execution time differences Change-Id: I0276e431d7d1ad89b1b28306cd0e0ea7e1174217 Implements: blueprint tempest-compare --- rally/cmd/api.py | 2 +- rally/cmd/commands/verify.py | 158 ++++++++---- .../verifiers/tempest/compare2html.py | 41 ++++ rally/verification/verifiers/tempest/diff.py | 108 +++++++++ .../tempest/report_templates/compare.mako | 165 +++++++++++++ tests/unit/cmd/commands/test_verify.py | 224 +++++++++++++----- .../verifiers/test_compare2html.py | 39 +++ .../unit/verification/verifiers/test_diff.py | 76 ++++++ tools/rally.bash_completion | 1 + 9 files changed, 703 insertions(+), 111 deletions(-) create mode 100644 rally/verification/verifiers/tempest/compare2html.py create mode 100644 rally/verification/verifiers/tempest/diff.py create mode 100644 rally/verification/verifiers/tempest/report_templates/compare.mako create mode 100644 tests/unit/verification/verifiers/test_compare2html.py create mode 100644 tests/unit/verification/verifiers/test_diff.py diff --git a/rally/cmd/api.py b/rally/cmd/api.py index 8b3482c63c..ce3649b8c3 100644 --- a/rally/cmd/api.py +++ b/rally/cmd/api.py @@ -32,7 +32,7 @@ LOG = log.getLogger(__name__) def main(): - # Initialize configuation and logging. + # Initialize configuration and logging. CONF(sys.argv[1:], project='rally') log.setup('rally') # Prepare application and bind to the service socket. diff --git a/rally/cmd/commands/verify.py b/rally/cmd/commands/verify.py index 38b8e0e9a7..036dfc89ff 100644 --- a/rally/cmd/commands/verify.py +++ b/rally/cmd/commands/verify.py @@ -15,6 +15,7 @@ """ Rally command: verify """ +import csv import json import os @@ -30,6 +31,7 @@ from rally.i18n import _ from rally import objects from rally.openstack.common import cliutils as common_cliutils from rally.orchestrator import api +from rally.verification.verifiers.tempest import diff from rally.verification.verifiers.tempest import json2html @@ -71,30 +73,30 @@ class VerifyCommands(object): return (1) verification = api.verify(deploy_id, set_name, regex, tempest_config) if do_use: - use.UseCommands().verification(verification['uuid']) + use.UseCommands().verification(verification["uuid"]) def list(self): """Display all verifications table, started and finished.""" - fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', 'Failures', - 'Created at', 'Status'] + fields = ["UUID", "Deployment UUID", "Set name", "Tests", "Failures", + "Created at", "Status"] verifications = db.verification_list() if verifications: common_cliutils.print_list(verifications, fields, - sortby_index=fields.index('Created at')) + sortby_index=fields.index("Created at")) else: print(_("There are no results from verifier. To run a verifier, " "use:\nrally verify start")) - @cliutils.args('--uuid', type=str, dest='verification_uuid', - help='UUID of the verification') - @cliutils.args('--html', action='store_true', dest='output_html', - help=('Results will be in html format')) - @cliutils.args('--json', action='store_true', dest='output_json', - help=('Results will be in json format')) - @cliutils.args('--output-file', type=str, required=False, - dest='output_file', - help='If specified, output will be saved to given file') + @cliutils.args("--uuid", type=str, dest="verification_uuid", + help="UUID of the verification") + @cliutils.args("--html", action="store_true", dest="output_html", + help=("Results will be in html format")) + @cliutils.args("--json", action="store_true", dest="output_json", + help=("Results will be in json format")) + @cliutils.args("--output-file", type=str, required=False, + dest="output_file", + help="If specified, output will be saved to given file") @envutils.with_default_verification_id def results(self, verification_uuid=None, output_file=None, output_html=None, output_json=None): @@ -107,13 +109,13 @@ class VerifyCommands(object): """ try: - results = db.verification_result_get(verification_uuid)['data'] + results = db.verification_result_get(verification_uuid)["data"] except exceptions.NotFoundException as e: print(six.text_type(e)) return 1 - result = '' - if len(filter(lambda x: bool(x), [output_json, output_html])) > 1: + result = "" + if output_json + output_html > 1: print("Please specify only one output format.") elif output_html: result = json2html.main(results) @@ -122,24 +124,24 @@ class VerifyCommands(object): if output_file: output_file = os.path.expanduser(output_file) - with open(output_file, 'wb') as f: + with open(output_file, "wb") as f: f.write(result) else: print(result) - @cliutils.args('--uuid', dest='verification_uuid', type=str, + @cliutils.args("--uuid", dest="verification_uuid", type=str, required=False, - help='UUID of a verification') - @cliutils.args('--sort-by', dest='sort_by', type=str, required=False, - help='Tests can be sorted by "name" or "duration"') - @cliutils.args('--detailed', dest='detailed', action='store_true', - required=False, help='Prints traceback of failed tests') + help="UUID of a verification") + @cliutils.args("--sort-by", dest="sort_by", type=str, required=False, + help="Tests can be sorted by 'name' or 'duration'") + @cliutils.args("--detailed", dest="detailed", action="store_true", + required=False, help="Prints traceback of failed tests") @envutils.with_default_verification_id - def show(self, verification_uuid=None, sort_by='name', detailed=False): + def show(self, verification_uuid=None, sort_by="name", detailed=False): """Display results table of the verification.""" try: - sortby_index = ('name', 'duration').index(sort_by) + sortby_index = ("name", "duration").index(sort_by) except ValueError: print("Sorry, but verification results can't be sorted " "by '%s'." % sort_by) @@ -153,42 +155,104 @@ class VerifyCommands(object): return 1 print ("Total results of verification:\n") - total_fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', - 'Failures', 'Created at', 'Status'] + total_fields = ["UUID", "Deployment UUID", "Set name", "Tests", + "Failures", "Created at", "Status"] common_cliutils.print_list([verification], fields=total_fields) print ("\nTests:\n") - fields = ['name', 'time', 'status'] + fields = ["name", "time", "status"] values = map(objects.Verification, - six.itervalues(tests.data['test_cases'])) + six.itervalues(tests.data["test_cases"])) common_cliutils.print_list(values, fields, sortby_index=sortby_index) if detailed: - for test in six.itervalues(tests.data['test_cases']): - if test['status'] == 'FAIL': + for test in six.itervalues(tests.data["test_cases"]): + if test["status"] == "FAIL": formatted_test = ( - '=====================================================' - '=================\n' - 'FAIL: %(name)s\n' - 'Time: %(time)s\n' - 'Type: %(type)s\n' - '-----------------------------------------------------' - '-----------------\n' - '%(log)s\n' + "=====================================================" + "=================\n" + "FAIL: %(name)s\n" + "Time: %(time)s\n" + "Type: %(type)s\n" + "-----------------------------------------------------" + "-----------------\n" + "%(log)s\n" ) % { - 'name': test['name'], 'time': test['time'], - 'type': test['failure']['type'], - 'log': test['failure']['log']} + "name": test["name"], "time": test["time"], + "type": test["failure"]["type"], + "log": test["failure"]["log"]} print (formatted_test) - @cliutils.args('--uuid', dest='verification_uuid', type=str, + @cliutils.args("--uuid", dest="verification_uuid", type=str, required=False, - help='UUID of a verification') - @cliutils.args('--sort-by', dest='sort_by', type=str, required=False, - help='Tests can be sorted by "name" or "duration"') + help="UUID of a verification") + @cliutils.args("--sort-by", dest="sort_by", type=str, required=False, + help="Tests can be sorted by 'name' or 'duration'") @envutils.with_default_verification_id - def detailed(self, verification_uuid=None, sort_by='name'): + def detailed(self, verification_uuid=None, sort_by="name"): """Display results table of verification with detailed errors.""" self.show(verification_uuid, sort_by, True) + + @cliutils.args("--uuid-1", type=str, dest="uuid1", + help="UUID of the first verification") + @cliutils.args("--uuid-2", type=str, dest="uuid2", + help="UUID of the second verification") + @cliutils.args("--csv", action="store_true", dest="output_csv", + help=("Save results in csv format to specified file")) + @cliutils.args("--html", action="store_true", dest="output_html", + help=("Save results in html format to specified file")) + @cliutils.args("--json", action="store_true", dest="output_json", + help=("Save results in json format to specified file")) + @cliutils.args("--output-file", type=str, required=False, + dest="output_file", + help="If specified, output will be saved to given file") + @cliutils.args("--threshold", type=int, required=False, + dest="threshold", default=0, + help="If specified, timing differences must exceed this " + "percentage threshold to be included in output") + def compare(self, uuid1=None, uuid2=None, + output_file=None, output_csv=None, output_html=None, + output_json=None, threshold=0): + """Compare two verification results. + + :param uuid1: First Verification UUID + :param uuid2: Second Verification UUID + :param output_file: If specified, output will be saved to given file + :param output_csv: Save results in csv format to the specified file + :param output_html: Save results in html format to the specified file + :param output_json: Save results in json format to the specified file + (Default) + :param threshold: Timing difference threshold percentage + """ + + try: + results1 = db.verification_result_get(uuid1)["data"]["test_cases"] + results2 = db.verification_result_get(uuid2)["data"]["test_cases"] + _diff = diff.Diff(results1, results2, threshold) + except exceptions.NotFoundException as e: + print(six.text_type(e)) + return 1 + + result = "" + if output_json + output_html + output_csv > 1: + print("Please specify only one output format, either --json, " + "--html or --csv.") + return 1 + elif output_html: + result = _diff.to_html() + elif output_csv: + result = _diff.to_csv() + else: + result = _diff.to_json() + + if output_file: + with open(output_file, "wb") as f: + if output_csv: + writer = csv.writer(f, dialect="excel") + writer.writerows(result) + else: + f.write(result) + else: + print(result) diff --git a/rally/verification/verifiers/tempest/compare2html.py b/rally/verification/verifiers/tempest/compare2html.py new file mode 100644 index 0000000000..1cd767691d --- /dev/null +++ b/rally/verification/verifiers/tempest/compare2html.py @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Output verification comparison results in html.""" + +import os + +import mako.template + +__description__ = "List differences between two verification runs" +__title__ = "Verification Comparison" +__version__ = "0.1" + + +def create_report(results): + template_kw = { + "heading": { + "title": __title__, + "description": __description__, + "parameters": [("Difference Count", len(results))] + }, + "generator": "compare2html %s" % __version__, + "results": results + } + + template_path = os.path.join(os.path.dirname(__file__), + "report_templates", + "compare.mako") + + with open(template_path) as f: + template = mako.template.Template(f.read(), strict_undefined=True) + output = template.render(**template_kw) + return output.encode('utf8') diff --git a/rally/verification/verifiers/tempest/diff.py b/rally/verification/verifiers/tempest/diff.py new file mode 100644 index 0000000000..993a3cb671 --- /dev/null +++ b/rally/verification/verifiers/tempest/diff.py @@ -0,0 +1,108 @@ +# Copyright 2014 Dell Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +import compare2html + + +class Diff(object): + + def __init__(self, test_cases1, test_cases2, threshold): + """Compare two verification results. + + Compares two verification results and emits + desired output, csv, html, json or pprint. + + :param test_cases1: older verification json + :param test_cases2: newer verification json + :param threshold: test time difference percentage threshold + + """ + self.threshold = threshold + self.diffs = self._compare(test_cases1, test_cases2) + + def _compare(self, tc1, tc2): + """Compare two verification results. + + :param tc1: first verification test cases json + :param tc2: second verification test cases json + + Typical test case json schema: + "test_case_key": { + "failure": { + "log": "" + }, + "name": "", + "output": "", + "status": "", + "time": 0.0 + } + """ + names1 = sorted(tc1) + names2 = sorted(tc2) + + diffs = [] + i = j = 0 + while i < len(names1) and j < len(names2): + name1 = names1[i] if i < len(names1) else None + name2 = names2[j] if j < len(names2) else None + if name1 and name2 and name1 == name2: + diffs.extend(self._diff_values(name1, tc1[name1], tc2[name2])) + i += 1 + j += 1 + + elif (not name1) or (name1 > name2): + diffs.append({"type": "new_test", "test_name": name2}) + j += 1 + else: + diffs.append({"type": "removed_test", "test_name": name1}) + i += 1 + + return diffs + + def _diff_values(self, name, result1, result2): + th = self.threshold + fields = ["status", "time", "output"] + diffs = [] + for field in fields: + val1 = result1[field] + val2 = result2[field] + if val1 != val2 and not (field == "time" + and abs(((val2 - val1) / val1) * 100) + < th): + diffs.append({ + "field": field, + "type": "value_changed", + "test_name": name, + "val1": val1, + "val2": val2 + }) + return diffs + + def to_csv(self): + rows = (("Type", "Field", "Value 1", "Value 2", "Test Name"),) + for res in self.diffs: + row = (res.get("type"), res.get("field", ""), + res.get("val1", ""), res.get("val2", ""), + res.get("test_name")) + rows = rows + (row,) + return rows + + def to_json(self): + return json.dumps(self.diffs, sort_keys=True, indent=4) + + def to_html(self): + return compare2html.create_report(self.diffs) diff --git a/rally/verification/verifiers/tempest/report_templates/compare.mako b/rally/verification/verifiers/tempest/report_templates/compare.mako new file mode 100644 index 0000000000..477cc3d4d5 --- /dev/null +++ b/rally/verification/verifiers/tempest/report_templates/compare.mako @@ -0,0 +1,165 @@ + + + + + ${heading["title"]} + + + + + + + +
+

${heading["title"]}

+ % for name, value in heading["parameters"]: +

${name}: ${value}

+ % endfor +

${heading["description"]}

+
+ + + + + + + + + + + + + + + + + + + + % for diff in results: + + + + + + + + % endfor +
TypeFieldValue 1Value 2Test Name
${diff.get("type")}${diff.get("field", "")}${diff.get("val1", "")}${diff.get("val2", "")}${diff.get("test_name")}
+ + + diff --git a/tests/unit/cmd/commands/test_verify.py b/tests/unit/cmd/commands/test_verify.py index 502e479a83..cb68c63872 100644 --- a/tests/unit/cmd/commands/test_verify.py +++ b/tests/unit/cmd/commands/test_verify.py @@ -31,41 +31,41 @@ class VerifyCommandsTestCase(test.TestCase): self.verify = verify.VerifyCommands() self.image1 = mock.Mock() - self.image1.name = 'cirros-1' - self.image1.id = 'fake_image_id_1' + self.image1.name = "cirros-1" + self.image1.id = "fake_image_id_1" self.image2 = mock.Mock() - self.image2.id = 'fake_image_id_2' - self.image2.name = 'cirros-2' + self.image2.id = "fake_image_id_2" + self.image2.name = "cirros-2" self.flavor1 = mock.Mock() self.flavor2 = mock.Mock() - self.flavor1.id = 'fake_flavor_id_1' - self.flavor2.id = 'fake_flavor_id_2' + self.flavor1.id = "fake_flavor_id_1" + self.flavor2.id = "fake_flavor_id_2" self.flavor1.ram = 128 self.flavor2.ram = 64 - @mock.patch('rally.osclients.Clients') - @mock.patch('rally.orchestrator.api.verify') + @mock.patch("rally.osclients.Clients") + @mock.patch("rally.orchestrator.api.verify") def test_start(self, mock_verify, mock_clients): - deploy_id = '0fba91c6-82d5-4ce1-bd00-5d7c989552d9' + deploy_id = "0fba91c6-82d5-4ce1-bd00-5d7c989552d9" mock_clients().glance().images.list.return_value = [ self.image1, self.image2] mock_clients().nova().flavors.list.return_value = [ self.flavor1, self.flavor2] self.verify.start(deploy_id=deploy_id) - default_set_name = 'smoke' + default_set_name = "smoke" default_regex = None mock_verify.assert_called_once_with(deploy_id, default_set_name, default_regex, None) - @mock.patch('rally.osclients.Clients') - @mock.patch('rally.orchestrator.api.verify') + @mock.patch("rally.osclients.Clients") + @mock.patch("rally.orchestrator.api.verify") def test_start_with_user_specified_tempest_config(self, mock_verify, mock_clients): - deploy_id = '0fba91c6-82d5-4ce1-bd00-5d7c989552d9' + deploy_id = "0fba91c6-82d5-4ce1-bd00-5d7c989552d9" mock_clients().glance().images.list.return_value = [ self.image1, self.image2] mock_clients().nova().flavors.list.return_value = [ @@ -73,7 +73,7 @@ class VerifyCommandsTestCase(test.TestCase): tempest_config = tempfile.NamedTemporaryFile() self.verify.start(deploy_id=deploy_id, tempest_config=tempest_config.name) - default_set_name = 'smoke' + default_set_name = "smoke" default_regex = None mock_verify.assert_called_once_with(deploy_id, @@ -81,114 +81,212 @@ class VerifyCommandsTestCase(test.TestCase): tempest_config.name) tempest_config.close() - @mock.patch('rally.orchestrator.api.verify') + @mock.patch("rally.orchestrator.api.verify") def test_start_with_wrong_set_name(self, mock_verify): - deploy_id = 'f2009aae-6ef3-468e-96b2-3c987d584010' + deploy_id = "f2009aae-6ef3-468e-96b2-3c987d584010" - wrong_set_name = 'unexpected_value' + wrong_set_name = "unexpected_value" self.verify.start(deploy_id, wrong_set_name) self.assertNotIn(wrong_set_name, consts.TEMPEST_TEST_SETS) self.assertFalse(mock_verify.called) - @mock.patch('rally.openstack.common.cliutils.print_list') - @mock.patch('rally.db.verification_list') + @mock.patch("rally.openstack.common.cliutils.print_list") + @mock.patch("rally.db.verification_list") def test_list(self, mock_db_verification_list, mock_print_list): - fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', 'Failures', - 'Created at', 'Status'] - verifications = {'dummy': []} + fields = ["UUID", "Deployment UUID", "Set name", "Tests", "Failures", + "Created at", "Status"] + verifications = {"dummy": []} mock_db_verification_list.return_value = verifications self.verify.list() mock_db_verification_list.assert_called_once_with() mock_print_list.assert_called_once_with(verifications, fields, sortby_index=fields.index( - 'Created at')) + "Created at")) - @mock.patch('rally.openstack.common.cliutils.print_list') - @mock.patch('rally.db.verification_get') - @mock.patch('rally.db.verification_result_get') - @mock.patch('rally.objects.Verification') + @mock.patch("rally.openstack.common.cliutils.print_list") + @mock.patch("rally.db.verification_get") + @mock.patch("rally.db.verification_result_get") + @mock.patch("rally.objects.Verification") def test_show(self, mock_obj_verification, mock_verification_result_get, mock_verification_get, mock_print_list): class Test_dummy(): - data = {'test_cases': {'test_a': {'name': 'test_a', 'time': 20, - 'status': 'PASS'}, - 'test_b': {'name': 'test_b', 'time': 20, - 'status': 'SKIP'}, - 'test_c': {'name': 'test_c', 'time': 20, - 'status': 'FAIL'}}} + data = {"test_cases": {"test_a": {"name": "test_a", "time": 20, + "status": "PASS"}, + "test_b": {"name": "test_b", "time": 20, + "status": "SKIP"}, + "test_c": {"name": "test_c", "time": 20, + "status": "FAIL"}}} - verification_id = '39121186-b9a4-421d-b094-6c6b270cf9e9' - total_fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', - 'Failures', 'Created at', 'Status'] - fields = ['name', 'time', 'status'] + verification_id = "39121186-b9a4-421d-b094-6c6b270cf9e9" + total_fields = ["UUID", "Deployment UUID", "Set name", "Tests", + "Failures", "Created at", "Status"] + fields = ["name", "time", "status"] verification = mock.MagicMock() tests = Test_dummy() mock_verification_result_get.return_value = tests mock_verification_get.return_value = verification mock_obj_verification.return_value = 1 values = map(objects.Verification, - six.itervalues(tests.data['test_cases'])) + six.itervalues(tests.data["test_cases"])) self.verify.show(verification_id) - mock_print_list.assert_any_call( - [verification], fields=total_fields) + mock_print_list.assert_any_call([verification], fields=total_fields) mock_verification_get.assert_called_once_with(verification_id) mock_verification_result_get.assert_called_once_with(verification_id) mock_print_list.assert_any_call(values, fields, sortby_index=0) - @mock.patch('rally.db.verification_result_get', return_value={'data': {}}) - @mock.patch('json.dumps') + @mock.patch("rally.db.verification_result_get", return_value={"data": {}}) + @mock.patch("json.dumps") def test_results(self, mock_json_dumps, mock_db_result_get): - verification_uuid = 'a0231bdf-6a4e-4daf-8ab1-ae076f75f070' - self.verify.results(verification_uuid, output_json=True) + verification_uuid = "a0231bdf-6a4e-4daf-8ab1-ae076f75f070" + self.verify.results(verification_uuid, output_html=False, + output_json=True) mock_db_result_get.assert_called_once_with(verification_uuid) mock_json_dumps.assert_called_once_with({}, sort_keys=True, indent=4) - @mock.patch('rally.db.verification_result_get') + @mock.patch("rally.db.verification_result_get") def test_results_verification_not_found(self, mock_db_result_get): - verification_uuid = '9044ced5-9c84-4666-8a8f-4b73a2b62acb' + verification_uuid = "9044ced5-9c84-4666-8a8f-4b73a2b62acb" mock_db_result_get.side_effect = exceptions.NotFoundException() - self.assertEqual(self.verify.results(verification_uuid), 1) + self.assertEqual(self.verify.results(verification_uuid, + output_html=False, + output_json=True), 1) mock_db_result_get.assert_called_once_with(verification_uuid) - @mock.patch('rally.cmd.commands.verify.open', create=True) - @mock.patch('rally.db.verification_result_get', return_value={'data': {}}) + @mock.patch("rally.cmd.commands.verify.open", create=True) + @mock.patch("rally.db.verification_result_get", return_value={"data": {}}) def test_results_with_output_json_and_output_file(self, mock_db_result_get, mock_open): mock_open.return_value = mock.MagicMock() - verification_uuid = '94615cd4-ff45-4123-86bd-4b0741541d09' - self.verify.results(verification_uuid, output_file='results', - output_json=True) + verification_uuid = "94615cd4-ff45-4123-86bd-4b0741541d09" + self.verify.results(verification_uuid, output_file="results", + output_html=False, output_json=True) mock_db_result_get.assert_called_once_with(verification_uuid) - mock_open.assert_called_once_with('results', 'wb') + mock_open.assert_called_once_with("results", "wb") fake_file = mock_open.return_value.__enter__.return_value - fake_file.write.assert_called_once_with('{}') + fake_file.write.assert_called_once_with("{}") - @mock.patch('rally.cmd.commands.verify.open', create=True) - @mock.patch('rally.db.verification_result_get') - @mock.patch('rally.verification.verifiers.tempest.json2html.main', - return_value='') + @mock.patch("rally.cmd.commands.verify.open", create=True) + @mock.patch("rally.db.verification_result_get") + @mock.patch("rally.verification.verifiers.tempest.json2html.main", + return_value="") def test_results_with_output_html_and_output_file(self, mock_json2html_main, mock_db_result_get, mock_open): mock_open.return_value = mock.MagicMock() - verification_uuid = '7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa' + verification_uuid = "7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa" fake_data = {} - results = {'data': fake_data} + results = {"data": fake_data} mock_db_result_get.return_value = results self.verify.results(verification_uuid, output_html=True, - output_file='results') + output_json=False, output_file="results") mock_db_result_get.assert_called_once_with(verification_uuid) mock_json2html_main.assert_called_once_with(fake_data) - mock_open.assert_called_once_with('results', 'wb') + mock_open.assert_called_once_with("results", "wb") fake_file = mock_open.return_value.__enter__.return_value - fake_file.write.assert_called_once_with('') + fake_file.write.assert_called_once_with("") + + @mock.patch("rally.db.verification_result_get", + return_value={"data": {"test_cases": {}}}) + @mock.patch("json.dumps") + def test_compare(self, mock_json_dumps, mock_db_result_get): + uuid1 = "8eda1b10-c8a4-4316-9603-8468ff1d1560" + uuid2 = "f6ef0a98-1b18-452f-a6a7-922555c2e326" + self.verify.compare(uuid1, uuid2, output_csv=False, output_html=False, + output_json=True) + + fake_data = [] + calls = [mock.call(uuid1), + mock.call(uuid2)] + mock_db_result_get.assert_has_calls(calls, True) + mock_json_dumps.assert_called_once_with(fake_data, sort_keys=True, + indent=4) + + @mock.patch("rally.db.verification_result_get", + side_effect=exceptions.NotFoundException()) + def test_compare_verification_not_found(self, mock_db_result_get): + uuid1 = "f7dc82da-31a6-4d40-bbf8-6d366d58960f" + uuid2 = "2f8a05f3-d310-4f02-aabf-e1165aaa5f9c" + + self.assertEqual(self.verify.compare(uuid1, uuid2, output_csv=False, + output_html=False, + output_json=True), 1) + + mock_db_result_get.assert_called_once_with(uuid1) + + @mock.patch("rally.cmd.commands.verify.open", create=True) + @mock.patch("rally.db.verification_result_get", + return_value={"data": {"test_cases": {}}}) + def test_compare_with_output_csv_and_output_file(self, + mock_db_result_get, + mock_open): + + fake_string = "Type,Field,Value 1,Value 2,Test Name\r\n" + uuid1 = "5e744557-4c3a-414f-9afb-7d3d8708028f" + uuid2 = "efe1c74d-a632-476e-bb6a-55a9aa9cf76b" + self.verify.compare(uuid1, uuid2, output_file="results", + output_csv=True, output_html=False, + output_json=False) + + calls = [mock.call(uuid1), + mock.call(uuid2)] + mock_db_result_get.assert_has_calls(calls, True) + mock_open.assert_called_once_with("results", "wb") + fake_file = mock_open.return_value.__enter__.return_value + fake_file.write.assert_called_once_with(fake_string) + + @mock.patch("rally.cmd.commands.verify.open", create=True) + @mock.patch("rally.db.verification_result_get", + return_value={"data": {"test_cases": {}}}) + def test_compare_with_output_json_and_output_file(self, + mock_db_result_get, + mock_open): + fake_json_string = "[]" + uuid1 = "0505e33a-738d-4474-a611-9db21547d863" + uuid2 = "b1908417-934e-481c-8d23-bc0badad39ed" + self.verify.compare(uuid1, uuid2, output_file="results", + output_csv=False, output_html=False, + output_json=True) + + calls = [mock.call(uuid1), + mock.call(uuid2)] + mock_db_result_get.assert_has_calls(calls, True) + mock_open.assert_called_once_with("results", "wb") + fake_file = mock_open.return_value.__enter__.return_value + fake_file.write.assert_called_once_with(fake_json_string) + + @mock.patch("rally.cmd.commands.verify.open", create=True) + @mock.patch("rally.db.verification_result_get") + @mock.patch(("rally.verification.verifiers.tempest." + "compare2html.create_report"), return_value="") + def test_compare_with_output_html_and_output_file(self, + mock_compare2html_create, + mock_db_result_get, + mock_open): + + uuid1 = "cdf64228-77e9-414d-9d4b-f65e9d62c61f" + uuid2 = "39393eec-1b45-4103-8ec1-631edac4b8f0" + results = {"data": {"test_cases": {}}} + fake_data = [] + self.verify.compare(uuid1, uuid2, + output_file="results", + output_csv=False, output_html=True, + output_json=False) + mock_db_result_get.return_value = results + calls = [mock.call(uuid1), + mock.call(uuid2)] + mock_db_result_get.assert_has_calls(calls, True) + mock_compare2html_create.assert_called_once_with(fake_data) + + mock_open.assert_called_once_with("results", "wb") + fake_file = mock_open.return_value.__enter__.return_value + fake_file.write.assert_called_once_with("") diff --git a/tests/unit/verification/verifiers/test_compare2html.py b/tests/unit/verification/verifiers/test_compare2html.py new file mode 100644 index 0000000000..c2d8754505 --- /dev/null +++ b/tests/unit/verification/verifiers/test_compare2html.py @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from rally.verification.verifiers.tempest import compare2html +from tests.unit import test + + +class Compare2HtmlTestCase(test.TestCase): + + def test_main(self): + results = [{'val2': 0.0111, 'field': u'time', 'val1': 0.0222, + 'type': 'CHANGED', 'test_name': u'test.one'}, + {'val2': 0.111, 'field': u'time', 'val1': 0.222, + 'type': 'CHANGED', 'test_name': u'test.two'}, + {'val2': 1.11, 'field': u'time', 'val1': 2.22, + 'type': 'CHANGED', 'test_name': u'test.three'}] + + fake_kw = {"heading": + {"title": compare2html.__title__, + "description": compare2html.__description__, + "parameters": [("Difference Count", len(results))] + }, + "generator": "compare2html %s" % compare2html.__version__, + "results": results} + + with mock.patch('mako.template.Template') as mock_mako: + compare2html.create_report(results) + mock_mako().render.assert_called_once_with(**fake_kw) diff --git a/tests/unit/verification/verifiers/test_diff.py b/tests/unit/verification/verifiers/test_diff.py new file mode 100644 index 0000000000..d623562b5d --- /dev/null +++ b/tests/unit/verification/verifiers/test_diff.py @@ -0,0 +1,76 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from rally.verification.verifiers.tempest import diff +from tests.unit import test + + +class DiffTestCase(test.TestCase): + + def test_main(self): + results1 = {'test.NONE': {'name': 'test.NONE', + 'output': 'test.NONE', + 'status': 'SKIPPED', + 'time': 0.000}, + 'test.one': {'name': 'test.one', + 'output': 'test.one', + 'status': 'OK', + 'time': 0.111}, + 'test.two': {'name': 'test.two', + 'output': 'test.two', + 'status': 'OK', + 'time': 0.222}, + 'test.three': {'name': 'test.three', + 'output': 'test.three', + 'status': 'FAILED', + 'time': 0.333}, + 'test.four': {'name': 'test.four', + 'output': 'test.four', + 'status': 'OK', + 'time': 0.444}, + 'test.five': {'name': 'test.five', + 'output': 'test.five', + 'status': 'OK', + 'time': 0.555} + } + + results2 = {'test.one': {'name': 'test.one', + 'output': 'test.one', + 'status': 'FAIL', + 'time': 0.1111}, + 'test.two': {'name': 'test.two', + 'output': 'test.two', + 'status': 'OK', + 'time': 0.222}, + 'test.three': {'name': 'test.three', + 'output': 'test.three', + 'status': 'OK', + 'time': 0.3333}, + 'test.four': {'name': 'test.four', + 'output': 'test.four', + 'status': 'FAIL', + 'time': 0.4444}, + 'test.five': {'name': 'test.five', + 'output': 'test.five', + 'status': 'OK', + 'time': 0.555}, + 'test.six': {'name': 'test.six', + 'output': 'test.six', + 'status': 'OK', + 'time': 0.666} + } + + diff_ = diff.Diff(results1, results2, 0) + assert len(diff_.diffs) == 8 + assert diff_.to_csv() != '' + assert diff_.to_html() != '' + assert diff_.to_json() != '' diff --git a/tools/rally.bash_completion b/tools/rally.bash_completion index af84cae65e..8bfa3d8612 100644 --- a/tools/rally.bash_completion +++ b/tools/rally.bash_completion @@ -27,6 +27,7 @@ _rally() OPTS["show_keypairs"]="--deploy-id" OPTS["show_networks"]="--deploy-id" OPTS["show_secgroups"]="--deploy-id" + OPTS["verify_compare"]="--uuid-1 --uuid-2 --csv --html --json --output-file --threshold" OPTS["verify_detailed"]="--uuid --sort-by" OPTS["verify_list"]="" OPTS["verify_results"]="--uuid --html --json --output-file"