diff --git a/rally/cmd/api.py b/rally/cmd/api.py index 8b3482c63c..ce3649b8c3 100644 --- a/rally/cmd/api.py +++ b/rally/cmd/api.py @@ -32,7 +32,7 @@ LOG = log.getLogger(__name__) def main(): - # Initialize configuation and logging. + # Initialize configuration and logging. CONF(sys.argv[1:], project='rally') log.setup('rally') # Prepare application and bind to the service socket. diff --git a/rally/cmd/commands/verify.py b/rally/cmd/commands/verify.py index 38b8e0e9a7..036dfc89ff 100644 --- a/rally/cmd/commands/verify.py +++ b/rally/cmd/commands/verify.py @@ -15,6 +15,7 @@ """ Rally command: verify """ +import csv import json import os @@ -30,6 +31,7 @@ from rally.i18n import _ from rally import objects from rally.openstack.common import cliutils as common_cliutils from rally.orchestrator import api +from rally.verification.verifiers.tempest import diff from rally.verification.verifiers.tempest import json2html @@ -71,30 +73,30 @@ class VerifyCommands(object): return (1) verification = api.verify(deploy_id, set_name, regex, tempest_config) if do_use: - use.UseCommands().verification(verification['uuid']) + use.UseCommands().verification(verification["uuid"]) def list(self): """Display all verifications table, started and finished.""" - fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', 'Failures', - 'Created at', 'Status'] + fields = ["UUID", "Deployment UUID", "Set name", "Tests", "Failures", + "Created at", "Status"] verifications = db.verification_list() if verifications: common_cliutils.print_list(verifications, fields, - sortby_index=fields.index('Created at')) + sortby_index=fields.index("Created at")) else: print(_("There are no results from verifier. To run a verifier, " "use:\nrally verify start")) - @cliutils.args('--uuid', type=str, dest='verification_uuid', - help='UUID of the verification') - @cliutils.args('--html', action='store_true', dest='output_html', - help=('Results will be in html format')) - @cliutils.args('--json', action='store_true', dest='output_json', - help=('Results will be in json format')) - @cliutils.args('--output-file', type=str, required=False, - dest='output_file', - help='If specified, output will be saved to given file') + @cliutils.args("--uuid", type=str, dest="verification_uuid", + help="UUID of the verification") + @cliutils.args("--html", action="store_true", dest="output_html", + help=("Results will be in html format")) + @cliutils.args("--json", action="store_true", dest="output_json", + help=("Results will be in json format")) + @cliutils.args("--output-file", type=str, required=False, + dest="output_file", + help="If specified, output will be saved to given file") @envutils.with_default_verification_id def results(self, verification_uuid=None, output_file=None, output_html=None, output_json=None): @@ -107,13 +109,13 @@ class VerifyCommands(object): """ try: - results = db.verification_result_get(verification_uuid)['data'] + results = db.verification_result_get(verification_uuid)["data"] except exceptions.NotFoundException as e: print(six.text_type(e)) return 1 - result = '' - if len(filter(lambda x: bool(x), [output_json, output_html])) > 1: + result = "" + if output_json + output_html > 1: print("Please specify only one output format.") elif output_html: result = json2html.main(results) @@ -122,24 +124,24 @@ class VerifyCommands(object): if output_file: output_file = os.path.expanduser(output_file) - with open(output_file, 'wb') as f: + with open(output_file, "wb") as f: f.write(result) else: print(result) - @cliutils.args('--uuid', dest='verification_uuid', type=str, + @cliutils.args("--uuid", dest="verification_uuid", type=str, required=False, - help='UUID of a verification') - @cliutils.args('--sort-by', dest='sort_by', type=str, required=False, - help='Tests can be sorted by "name" or "duration"') - @cliutils.args('--detailed', dest='detailed', action='store_true', - required=False, help='Prints traceback of failed tests') + help="UUID of a verification") + @cliutils.args("--sort-by", dest="sort_by", type=str, required=False, + help="Tests can be sorted by 'name' or 'duration'") + @cliutils.args("--detailed", dest="detailed", action="store_true", + required=False, help="Prints traceback of failed tests") @envutils.with_default_verification_id - def show(self, verification_uuid=None, sort_by='name', detailed=False): + def show(self, verification_uuid=None, sort_by="name", detailed=False): """Display results table of the verification.""" try: - sortby_index = ('name', 'duration').index(sort_by) + sortby_index = ("name", "duration").index(sort_by) except ValueError: print("Sorry, but verification results can't be sorted " "by '%s'." % sort_by) @@ -153,42 +155,104 @@ class VerifyCommands(object): return 1 print ("Total results of verification:\n") - total_fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', - 'Failures', 'Created at', 'Status'] + total_fields = ["UUID", "Deployment UUID", "Set name", "Tests", + "Failures", "Created at", "Status"] common_cliutils.print_list([verification], fields=total_fields) print ("\nTests:\n") - fields = ['name', 'time', 'status'] + fields = ["name", "time", "status"] values = map(objects.Verification, - six.itervalues(tests.data['test_cases'])) + six.itervalues(tests.data["test_cases"])) common_cliutils.print_list(values, fields, sortby_index=sortby_index) if detailed: - for test in six.itervalues(tests.data['test_cases']): - if test['status'] == 'FAIL': + for test in six.itervalues(tests.data["test_cases"]): + if test["status"] == "FAIL": formatted_test = ( - '=====================================================' - '=================\n' - 'FAIL: %(name)s\n' - 'Time: %(time)s\n' - 'Type: %(type)s\n' - '-----------------------------------------------------' - '-----------------\n' - '%(log)s\n' + "=====================================================" + "=================\n" + "FAIL: %(name)s\n" + "Time: %(time)s\n" + "Type: %(type)s\n" + "-----------------------------------------------------" + "-----------------\n" + "%(log)s\n" ) % { - 'name': test['name'], 'time': test['time'], - 'type': test['failure']['type'], - 'log': test['failure']['log']} + "name": test["name"], "time": test["time"], + "type": test["failure"]["type"], + "log": test["failure"]["log"]} print (formatted_test) - @cliutils.args('--uuid', dest='verification_uuid', type=str, + @cliutils.args("--uuid", dest="verification_uuid", type=str, required=False, - help='UUID of a verification') - @cliutils.args('--sort-by', dest='sort_by', type=str, required=False, - help='Tests can be sorted by "name" or "duration"') + help="UUID of a verification") + @cliutils.args("--sort-by", dest="sort_by", type=str, required=False, + help="Tests can be sorted by 'name' or 'duration'") @envutils.with_default_verification_id - def detailed(self, verification_uuid=None, sort_by='name'): + def detailed(self, verification_uuid=None, sort_by="name"): """Display results table of verification with detailed errors.""" self.show(verification_uuid, sort_by, True) + + @cliutils.args("--uuid-1", type=str, dest="uuid1", + help="UUID of the first verification") + @cliutils.args("--uuid-2", type=str, dest="uuid2", + help="UUID of the second verification") + @cliutils.args("--csv", action="store_true", dest="output_csv", + help=("Save results in csv format to specified file")) + @cliutils.args("--html", action="store_true", dest="output_html", + help=("Save results in html format to specified file")) + @cliutils.args("--json", action="store_true", dest="output_json", + help=("Save results in json format to specified file")) + @cliutils.args("--output-file", type=str, required=False, + dest="output_file", + help="If specified, output will be saved to given file") + @cliutils.args("--threshold", type=int, required=False, + dest="threshold", default=0, + help="If specified, timing differences must exceed this " + "percentage threshold to be included in output") + def compare(self, uuid1=None, uuid2=None, + output_file=None, output_csv=None, output_html=None, + output_json=None, threshold=0): + """Compare two verification results. + + :param uuid1: First Verification UUID + :param uuid2: Second Verification UUID + :param output_file: If specified, output will be saved to given file + :param output_csv: Save results in csv format to the specified file + :param output_html: Save results in html format to the specified file + :param output_json: Save results in json format to the specified file + (Default) + :param threshold: Timing difference threshold percentage + """ + + try: + results1 = db.verification_result_get(uuid1)["data"]["test_cases"] + results2 = db.verification_result_get(uuid2)["data"]["test_cases"] + _diff = diff.Diff(results1, results2, threshold) + except exceptions.NotFoundException as e: + print(six.text_type(e)) + return 1 + + result = "" + if output_json + output_html + output_csv > 1: + print("Please specify only one output format, either --json, " + "--html or --csv.") + return 1 + elif output_html: + result = _diff.to_html() + elif output_csv: + result = _diff.to_csv() + else: + result = _diff.to_json() + + if output_file: + with open(output_file, "wb") as f: + if output_csv: + writer = csv.writer(f, dialect="excel") + writer.writerows(result) + else: + f.write(result) + else: + print(result) diff --git a/rally/verification/verifiers/tempest/compare2html.py b/rally/verification/verifiers/tempest/compare2html.py new file mode 100644 index 0000000000..1cd767691d --- /dev/null +++ b/rally/verification/verifiers/tempest/compare2html.py @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Output verification comparison results in html.""" + +import os + +import mako.template + +__description__ = "List differences between two verification runs" +__title__ = "Verification Comparison" +__version__ = "0.1" + + +def create_report(results): + template_kw = { + "heading": { + "title": __title__, + "description": __description__, + "parameters": [("Difference Count", len(results))] + }, + "generator": "compare2html %s" % __version__, + "results": results + } + + template_path = os.path.join(os.path.dirname(__file__), + "report_templates", + "compare.mako") + + with open(template_path) as f: + template = mako.template.Template(f.read(), strict_undefined=True) + output = template.render(**template_kw) + return output.encode('utf8') diff --git a/rally/verification/verifiers/tempest/diff.py b/rally/verification/verifiers/tempest/diff.py new file mode 100644 index 0000000000..993a3cb671 --- /dev/null +++ b/rally/verification/verifiers/tempest/diff.py @@ -0,0 +1,108 @@ +# Copyright 2014 Dell Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +import compare2html + + +class Diff(object): + + def __init__(self, test_cases1, test_cases2, threshold): + """Compare two verification results. + + Compares two verification results and emits + desired output, csv, html, json or pprint. + + :param test_cases1: older verification json + :param test_cases2: newer verification json + :param threshold: test time difference percentage threshold + + """ + self.threshold = threshold + self.diffs = self._compare(test_cases1, test_cases2) + + def _compare(self, tc1, tc2): + """Compare two verification results. + + :param tc1: first verification test cases json + :param tc2: second verification test cases json + + Typical test case json schema: + "test_case_key": { + "failure": { + "log": "" + }, + "name": "", + "output": "", + "status": "", + "time": 0.0 + } + """ + names1 = sorted(tc1) + names2 = sorted(tc2) + + diffs = [] + i = j = 0 + while i < len(names1) and j < len(names2): + name1 = names1[i] if i < len(names1) else None + name2 = names2[j] if j < len(names2) else None + if name1 and name2 and name1 == name2: + diffs.extend(self._diff_values(name1, tc1[name1], tc2[name2])) + i += 1 + j += 1 + + elif (not name1) or (name1 > name2): + diffs.append({"type": "new_test", "test_name": name2}) + j += 1 + else: + diffs.append({"type": "removed_test", "test_name": name1}) + i += 1 + + return diffs + + def _diff_values(self, name, result1, result2): + th = self.threshold + fields = ["status", "time", "output"] + diffs = [] + for field in fields: + val1 = result1[field] + val2 = result2[field] + if val1 != val2 and not (field == "time" + and abs(((val2 - val1) / val1) * 100) + < th): + diffs.append({ + "field": field, + "type": "value_changed", + "test_name": name, + "val1": val1, + "val2": val2 + }) + return diffs + + def to_csv(self): + rows = (("Type", "Field", "Value 1", "Value 2", "Test Name"),) + for res in self.diffs: + row = (res.get("type"), res.get("field", ""), + res.get("val1", ""), res.get("val2", ""), + res.get("test_name")) + rows = rows + (row,) + return rows + + def to_json(self): + return json.dumps(self.diffs, sort_keys=True, indent=4) + + def to_html(self): + return compare2html.create_report(self.diffs) diff --git a/rally/verification/verifiers/tempest/report_templates/compare.mako b/rally/verification/verifiers/tempest/report_templates/compare.mako new file mode 100644 index 0000000000..477cc3d4d5 --- /dev/null +++ b/rally/verification/verifiers/tempest/report_templates/compare.mako @@ -0,0 +1,165 @@ + + + + + ${heading["title"]} + + + + + + + +
+

${heading["title"]}

+ % for name, value in heading["parameters"]: +

${name}: ${value}

+ % endfor +

${heading["description"]}

+
+ + + + + + + + + + + + + + + + + + + + % for diff in results: + + + + + + + + % endfor +
TypeFieldValue 1Value 2Test Name
${diff.get("type")}${diff.get("field", "")}${diff.get("val1", "")}${diff.get("val2", "")}${diff.get("test_name")}
+ + + diff --git a/tests/unit/cmd/commands/test_verify.py b/tests/unit/cmd/commands/test_verify.py index 502e479a83..cb68c63872 100644 --- a/tests/unit/cmd/commands/test_verify.py +++ b/tests/unit/cmd/commands/test_verify.py @@ -31,41 +31,41 @@ class VerifyCommandsTestCase(test.TestCase): self.verify = verify.VerifyCommands() self.image1 = mock.Mock() - self.image1.name = 'cirros-1' - self.image1.id = 'fake_image_id_1' + self.image1.name = "cirros-1" + self.image1.id = "fake_image_id_1" self.image2 = mock.Mock() - self.image2.id = 'fake_image_id_2' - self.image2.name = 'cirros-2' + self.image2.id = "fake_image_id_2" + self.image2.name = "cirros-2" self.flavor1 = mock.Mock() self.flavor2 = mock.Mock() - self.flavor1.id = 'fake_flavor_id_1' - self.flavor2.id = 'fake_flavor_id_2' + self.flavor1.id = "fake_flavor_id_1" + self.flavor2.id = "fake_flavor_id_2" self.flavor1.ram = 128 self.flavor2.ram = 64 - @mock.patch('rally.osclients.Clients') - @mock.patch('rally.orchestrator.api.verify') + @mock.patch("rally.osclients.Clients") + @mock.patch("rally.orchestrator.api.verify") def test_start(self, mock_verify, mock_clients): - deploy_id = '0fba91c6-82d5-4ce1-bd00-5d7c989552d9' + deploy_id = "0fba91c6-82d5-4ce1-bd00-5d7c989552d9" mock_clients().glance().images.list.return_value = [ self.image1, self.image2] mock_clients().nova().flavors.list.return_value = [ self.flavor1, self.flavor2] self.verify.start(deploy_id=deploy_id) - default_set_name = 'smoke' + default_set_name = "smoke" default_regex = None mock_verify.assert_called_once_with(deploy_id, default_set_name, default_regex, None) - @mock.patch('rally.osclients.Clients') - @mock.patch('rally.orchestrator.api.verify') + @mock.patch("rally.osclients.Clients") + @mock.patch("rally.orchestrator.api.verify") def test_start_with_user_specified_tempest_config(self, mock_verify, mock_clients): - deploy_id = '0fba91c6-82d5-4ce1-bd00-5d7c989552d9' + deploy_id = "0fba91c6-82d5-4ce1-bd00-5d7c989552d9" mock_clients().glance().images.list.return_value = [ self.image1, self.image2] mock_clients().nova().flavors.list.return_value = [ @@ -73,7 +73,7 @@ class VerifyCommandsTestCase(test.TestCase): tempest_config = tempfile.NamedTemporaryFile() self.verify.start(deploy_id=deploy_id, tempest_config=tempest_config.name) - default_set_name = 'smoke' + default_set_name = "smoke" default_regex = None mock_verify.assert_called_once_with(deploy_id, @@ -81,114 +81,212 @@ class VerifyCommandsTestCase(test.TestCase): tempest_config.name) tempest_config.close() - @mock.patch('rally.orchestrator.api.verify') + @mock.patch("rally.orchestrator.api.verify") def test_start_with_wrong_set_name(self, mock_verify): - deploy_id = 'f2009aae-6ef3-468e-96b2-3c987d584010' + deploy_id = "f2009aae-6ef3-468e-96b2-3c987d584010" - wrong_set_name = 'unexpected_value' + wrong_set_name = "unexpected_value" self.verify.start(deploy_id, wrong_set_name) self.assertNotIn(wrong_set_name, consts.TEMPEST_TEST_SETS) self.assertFalse(mock_verify.called) - @mock.patch('rally.openstack.common.cliutils.print_list') - @mock.patch('rally.db.verification_list') + @mock.patch("rally.openstack.common.cliutils.print_list") + @mock.patch("rally.db.verification_list") def test_list(self, mock_db_verification_list, mock_print_list): - fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', 'Failures', - 'Created at', 'Status'] - verifications = {'dummy': []} + fields = ["UUID", "Deployment UUID", "Set name", "Tests", "Failures", + "Created at", "Status"] + verifications = {"dummy": []} mock_db_verification_list.return_value = verifications self.verify.list() mock_db_verification_list.assert_called_once_with() mock_print_list.assert_called_once_with(verifications, fields, sortby_index=fields.index( - 'Created at')) + "Created at")) - @mock.patch('rally.openstack.common.cliutils.print_list') - @mock.patch('rally.db.verification_get') - @mock.patch('rally.db.verification_result_get') - @mock.patch('rally.objects.Verification') + @mock.patch("rally.openstack.common.cliutils.print_list") + @mock.patch("rally.db.verification_get") + @mock.patch("rally.db.verification_result_get") + @mock.patch("rally.objects.Verification") def test_show(self, mock_obj_verification, mock_verification_result_get, mock_verification_get, mock_print_list): class Test_dummy(): - data = {'test_cases': {'test_a': {'name': 'test_a', 'time': 20, - 'status': 'PASS'}, - 'test_b': {'name': 'test_b', 'time': 20, - 'status': 'SKIP'}, - 'test_c': {'name': 'test_c', 'time': 20, - 'status': 'FAIL'}}} + data = {"test_cases": {"test_a": {"name": "test_a", "time": 20, + "status": "PASS"}, + "test_b": {"name": "test_b", "time": 20, + "status": "SKIP"}, + "test_c": {"name": "test_c", "time": 20, + "status": "FAIL"}}} - verification_id = '39121186-b9a4-421d-b094-6c6b270cf9e9' - total_fields = ['UUID', 'Deployment UUID', 'Set name', 'Tests', - 'Failures', 'Created at', 'Status'] - fields = ['name', 'time', 'status'] + verification_id = "39121186-b9a4-421d-b094-6c6b270cf9e9" + total_fields = ["UUID", "Deployment UUID", "Set name", "Tests", + "Failures", "Created at", "Status"] + fields = ["name", "time", "status"] verification = mock.MagicMock() tests = Test_dummy() mock_verification_result_get.return_value = tests mock_verification_get.return_value = verification mock_obj_verification.return_value = 1 values = map(objects.Verification, - six.itervalues(tests.data['test_cases'])) + six.itervalues(tests.data["test_cases"])) self.verify.show(verification_id) - mock_print_list.assert_any_call( - [verification], fields=total_fields) + mock_print_list.assert_any_call([verification], fields=total_fields) mock_verification_get.assert_called_once_with(verification_id) mock_verification_result_get.assert_called_once_with(verification_id) mock_print_list.assert_any_call(values, fields, sortby_index=0) - @mock.patch('rally.db.verification_result_get', return_value={'data': {}}) - @mock.patch('json.dumps') + @mock.patch("rally.db.verification_result_get", return_value={"data": {}}) + @mock.patch("json.dumps") def test_results(self, mock_json_dumps, mock_db_result_get): - verification_uuid = 'a0231bdf-6a4e-4daf-8ab1-ae076f75f070' - self.verify.results(verification_uuid, output_json=True) + verification_uuid = "a0231bdf-6a4e-4daf-8ab1-ae076f75f070" + self.verify.results(verification_uuid, output_html=False, + output_json=True) mock_db_result_get.assert_called_once_with(verification_uuid) mock_json_dumps.assert_called_once_with({}, sort_keys=True, indent=4) - @mock.patch('rally.db.verification_result_get') + @mock.patch("rally.db.verification_result_get") def test_results_verification_not_found(self, mock_db_result_get): - verification_uuid = '9044ced5-9c84-4666-8a8f-4b73a2b62acb' + verification_uuid = "9044ced5-9c84-4666-8a8f-4b73a2b62acb" mock_db_result_get.side_effect = exceptions.NotFoundException() - self.assertEqual(self.verify.results(verification_uuid), 1) + self.assertEqual(self.verify.results(verification_uuid, + output_html=False, + output_json=True), 1) mock_db_result_get.assert_called_once_with(verification_uuid) - @mock.patch('rally.cmd.commands.verify.open', create=True) - @mock.patch('rally.db.verification_result_get', return_value={'data': {}}) + @mock.patch("rally.cmd.commands.verify.open", create=True) + @mock.patch("rally.db.verification_result_get", return_value={"data": {}}) def test_results_with_output_json_and_output_file(self, mock_db_result_get, mock_open): mock_open.return_value = mock.MagicMock() - verification_uuid = '94615cd4-ff45-4123-86bd-4b0741541d09' - self.verify.results(verification_uuid, output_file='results', - output_json=True) + verification_uuid = "94615cd4-ff45-4123-86bd-4b0741541d09" + self.verify.results(verification_uuid, output_file="results", + output_html=False, output_json=True) mock_db_result_get.assert_called_once_with(verification_uuid) - mock_open.assert_called_once_with('results', 'wb') + mock_open.assert_called_once_with("results", "wb") fake_file = mock_open.return_value.__enter__.return_value - fake_file.write.assert_called_once_with('{}') + fake_file.write.assert_called_once_with("{}") - @mock.patch('rally.cmd.commands.verify.open', create=True) - @mock.patch('rally.db.verification_result_get') - @mock.patch('rally.verification.verifiers.tempest.json2html.main', - return_value='') + @mock.patch("rally.cmd.commands.verify.open", create=True) + @mock.patch("rally.db.verification_result_get") + @mock.patch("rally.verification.verifiers.tempest.json2html.main", + return_value="") def test_results_with_output_html_and_output_file(self, mock_json2html_main, mock_db_result_get, mock_open): mock_open.return_value = mock.MagicMock() - verification_uuid = '7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa' + verification_uuid = "7140dd59-3a7b-41fd-a3ef-5e3e615d7dfa" fake_data = {} - results = {'data': fake_data} + results = {"data": fake_data} mock_db_result_get.return_value = results self.verify.results(verification_uuid, output_html=True, - output_file='results') + output_json=False, output_file="results") mock_db_result_get.assert_called_once_with(verification_uuid) mock_json2html_main.assert_called_once_with(fake_data) - mock_open.assert_called_once_with('results', 'wb') + mock_open.assert_called_once_with("results", "wb") fake_file = mock_open.return_value.__enter__.return_value - fake_file.write.assert_called_once_with('') + fake_file.write.assert_called_once_with("") + + @mock.patch("rally.db.verification_result_get", + return_value={"data": {"test_cases": {}}}) + @mock.patch("json.dumps") + def test_compare(self, mock_json_dumps, mock_db_result_get): + uuid1 = "8eda1b10-c8a4-4316-9603-8468ff1d1560" + uuid2 = "f6ef0a98-1b18-452f-a6a7-922555c2e326" + self.verify.compare(uuid1, uuid2, output_csv=False, output_html=False, + output_json=True) + + fake_data = [] + calls = [mock.call(uuid1), + mock.call(uuid2)] + mock_db_result_get.assert_has_calls(calls, True) + mock_json_dumps.assert_called_once_with(fake_data, sort_keys=True, + indent=4) + + @mock.patch("rally.db.verification_result_get", + side_effect=exceptions.NotFoundException()) + def test_compare_verification_not_found(self, mock_db_result_get): + uuid1 = "f7dc82da-31a6-4d40-bbf8-6d366d58960f" + uuid2 = "2f8a05f3-d310-4f02-aabf-e1165aaa5f9c" + + self.assertEqual(self.verify.compare(uuid1, uuid2, output_csv=False, + output_html=False, + output_json=True), 1) + + mock_db_result_get.assert_called_once_with(uuid1) + + @mock.patch("rally.cmd.commands.verify.open", create=True) + @mock.patch("rally.db.verification_result_get", + return_value={"data": {"test_cases": {}}}) + def test_compare_with_output_csv_and_output_file(self, + mock_db_result_get, + mock_open): + + fake_string = "Type,Field,Value 1,Value 2,Test Name\r\n" + uuid1 = "5e744557-4c3a-414f-9afb-7d3d8708028f" + uuid2 = "efe1c74d-a632-476e-bb6a-55a9aa9cf76b" + self.verify.compare(uuid1, uuid2, output_file="results", + output_csv=True, output_html=False, + output_json=False) + + calls = [mock.call(uuid1), + mock.call(uuid2)] + mock_db_result_get.assert_has_calls(calls, True) + mock_open.assert_called_once_with("results", "wb") + fake_file = mock_open.return_value.__enter__.return_value + fake_file.write.assert_called_once_with(fake_string) + + @mock.patch("rally.cmd.commands.verify.open", create=True) + @mock.patch("rally.db.verification_result_get", + return_value={"data": {"test_cases": {}}}) + def test_compare_with_output_json_and_output_file(self, + mock_db_result_get, + mock_open): + fake_json_string = "[]" + uuid1 = "0505e33a-738d-4474-a611-9db21547d863" + uuid2 = "b1908417-934e-481c-8d23-bc0badad39ed" + self.verify.compare(uuid1, uuid2, output_file="results", + output_csv=False, output_html=False, + output_json=True) + + calls = [mock.call(uuid1), + mock.call(uuid2)] + mock_db_result_get.assert_has_calls(calls, True) + mock_open.assert_called_once_with("results", "wb") + fake_file = mock_open.return_value.__enter__.return_value + fake_file.write.assert_called_once_with(fake_json_string) + + @mock.patch("rally.cmd.commands.verify.open", create=True) + @mock.patch("rally.db.verification_result_get") + @mock.patch(("rally.verification.verifiers.tempest." + "compare2html.create_report"), return_value="") + def test_compare_with_output_html_and_output_file(self, + mock_compare2html_create, + mock_db_result_get, + mock_open): + + uuid1 = "cdf64228-77e9-414d-9d4b-f65e9d62c61f" + uuid2 = "39393eec-1b45-4103-8ec1-631edac4b8f0" + results = {"data": {"test_cases": {}}} + fake_data = [] + self.verify.compare(uuid1, uuid2, + output_file="results", + output_csv=False, output_html=True, + output_json=False) + mock_db_result_get.return_value = results + calls = [mock.call(uuid1), + mock.call(uuid2)] + mock_db_result_get.assert_has_calls(calls, True) + mock_compare2html_create.assert_called_once_with(fake_data) + + mock_open.assert_called_once_with("results", "wb") + fake_file = mock_open.return_value.__enter__.return_value + fake_file.write.assert_called_once_with("") diff --git a/tests/unit/verification/verifiers/test_compare2html.py b/tests/unit/verification/verifiers/test_compare2html.py new file mode 100644 index 0000000000..c2d8754505 --- /dev/null +++ b/tests/unit/verification/verifiers/test_compare2html.py @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from rally.verification.verifiers.tempest import compare2html +from tests.unit import test + + +class Compare2HtmlTestCase(test.TestCase): + + def test_main(self): + results = [{'val2': 0.0111, 'field': u'time', 'val1': 0.0222, + 'type': 'CHANGED', 'test_name': u'test.one'}, + {'val2': 0.111, 'field': u'time', 'val1': 0.222, + 'type': 'CHANGED', 'test_name': u'test.two'}, + {'val2': 1.11, 'field': u'time', 'val1': 2.22, + 'type': 'CHANGED', 'test_name': u'test.three'}] + + fake_kw = {"heading": + {"title": compare2html.__title__, + "description": compare2html.__description__, + "parameters": [("Difference Count", len(results))] + }, + "generator": "compare2html %s" % compare2html.__version__, + "results": results} + + with mock.patch('mako.template.Template') as mock_mako: + compare2html.create_report(results) + mock_mako().render.assert_called_once_with(**fake_kw) diff --git a/tests/unit/verification/verifiers/test_diff.py b/tests/unit/verification/verifiers/test_diff.py new file mode 100644 index 0000000000..d623562b5d --- /dev/null +++ b/tests/unit/verification/verifiers/test_diff.py @@ -0,0 +1,76 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from rally.verification.verifiers.tempest import diff +from tests.unit import test + + +class DiffTestCase(test.TestCase): + + def test_main(self): + results1 = {'test.NONE': {'name': 'test.NONE', + 'output': 'test.NONE', + 'status': 'SKIPPED', + 'time': 0.000}, + 'test.one': {'name': 'test.one', + 'output': 'test.one', + 'status': 'OK', + 'time': 0.111}, + 'test.two': {'name': 'test.two', + 'output': 'test.two', + 'status': 'OK', + 'time': 0.222}, + 'test.three': {'name': 'test.three', + 'output': 'test.three', + 'status': 'FAILED', + 'time': 0.333}, + 'test.four': {'name': 'test.four', + 'output': 'test.four', + 'status': 'OK', + 'time': 0.444}, + 'test.five': {'name': 'test.five', + 'output': 'test.five', + 'status': 'OK', + 'time': 0.555} + } + + results2 = {'test.one': {'name': 'test.one', + 'output': 'test.one', + 'status': 'FAIL', + 'time': 0.1111}, + 'test.two': {'name': 'test.two', + 'output': 'test.two', + 'status': 'OK', + 'time': 0.222}, + 'test.three': {'name': 'test.three', + 'output': 'test.three', + 'status': 'OK', + 'time': 0.3333}, + 'test.four': {'name': 'test.four', + 'output': 'test.four', + 'status': 'FAIL', + 'time': 0.4444}, + 'test.five': {'name': 'test.five', + 'output': 'test.five', + 'status': 'OK', + 'time': 0.555}, + 'test.six': {'name': 'test.six', + 'output': 'test.six', + 'status': 'OK', + 'time': 0.666} + } + + diff_ = diff.Diff(results1, results2, 0) + assert len(diff_.diffs) == 8 + assert diff_.to_csv() != '' + assert diff_.to_html() != '' + assert diff_.to_json() != '' diff --git a/tools/rally.bash_completion b/tools/rally.bash_completion index c050ab5eb3..9bce6982bf 100644 --- a/tools/rally.bash_completion +++ b/tools/rally.bash_completion @@ -31,6 +31,7 @@ _rally() OPTS["show_keypairs"]="--deploy-id" OPTS["show_networks"]="--deploy-id" OPTS["show_secgroups"]="--deploy-id" + OPTS["verify_compare"]="--uuid-1 --uuid-2 --csv --html --json --output-file --threshold" OPTS["verify_detailed"]="--uuid --sort-by" OPTS["verify_list"]="" OPTS["verify_results"]="--uuid --html --json --output-file" @@ -74,4 +75,4 @@ _rally() fi return 0 } -complete -F _rally rally \ No newline at end of file +complete -F _rally rally