Make verification reporter pluggable

Some time ago we added new entity "exporters". For task we created a command
"rally task export" which allows to export task results to external systems.
Exporters can "easily" extende by plugins.

In case of verification component, proper command wasn't created yet. While I
thing idea of exporters is good enough, implementation can be improved.
To simplify usage, entity "exporters" was renamed to "reporters" and
integrated with "rally verify report" command. Generation of regular rally
reports (like html, json) is done in the same way as in plugabble reporters.

Proposed interface:

  rally verify report --uuid <uuid-1> [<uuid-2>...<uuid-n>] \
      --type <reporter-name> --to <destination>

Examples of usage:

  rally verify report --uuid some-uuid --type html --to /path/to/save

  # such exporter is not implemented yet, but we expect it soon
  rally verify report --uuids some-uuid --type elasticsearch \
      --to https://username@passwd:example.com

Change-Id: I4fb38984a73f92503bf2988e509477b10b308cac

etc/rally.bash_completion

@@ -54,7 +54,7 @@ _rally()
     OPTS["verify_list-verifier-exts"]="--id"
     OPTS["verify_list-verifier-tests"]="--id --pattern"
     OPTS["verify_list-verifiers"]="--status"
-    OPTS["verify_report"]="--uuid --html --file --open"
+    OPTS["verify_report"]="--uuid --type --to --open"
     OPTS["verify_rerun"]="--uuid --deployment-id --failed"
     OPTS["verify_show"]="--uuid --sort-by --detailed"
     OPTS["verify_start"]="--verifier-id --deployment-id --pattern --concurrency --load-list --skip-list --xfail-list --no-use"

rally/api.py

@@ -35,9 +35,9 @@ from rally.deployment import engine as deploy_engine
 from rally import exceptions
 from rally import osclients
 from rally.task import engine
-from rally.ui import report
 from rally.verification import context as vcontext
 from rally.verification import manager as vmanager
+from rally.verification import reporter as vreporter
 
 
 CONF = cfg.CONF
@@ -895,18 +895,25 @@ class _Verification(object):
         LOG.info("Verification has been successfully deleted!")
 
     @classmethod
-    def report(cls, uuids, html=False):
+    def report(cls, uuids, output_type, output_dest=None):
         """Generate a report for a verification or a few verifications.
 
         :param uuids: List of verifications UUIDs
-        :param html: Whether or not to create the report in HTML format
+        :param output_type: Plugin name of verification reporter
+        :param output_dest: Destination for verification report
         """
         verifications = [cls.get(uuid) for uuid in uuids]
 
-        if html:
-            return report.VerificationReport(verifications).to_html()
+        reporter_cls = vreporter.VerificationReporter.get(output_type)
+        reporter_cls.validate(output_dest)
 
-        return report.VerificationReport(verifications).to_json()
+        LOG.info("Building '%s' report for the following verification(s): "
+                 "'%s'.", output_type, "', '".join(uuids))
+        result = vreporter.VerificationReporter.make(reporter_cls,
+                                                     verifications,
+                                                     output_dest)
+        LOG.info(_LI("The report has been successfully built."))
+        return result
 
     @classmethod
     def import_results(cls, verifier_id, deployment_id, data, **run_args):

rally/cli/commands/verify.py

@@ -39,6 +39,8 @@ LIST_DEPLOYMENTS_HINT = ("HINT: You can list all deployments, executing "
 LIST_VERIFICATIONS_HINT = ("HINT: You can list all verifications, executing "
                            "command `rally verify list`.")
 
+DEFAULTS_REPORTERS = ("HTML", "JSON")
+
 
 class VerifyCommands(object):
     """Verify an OpenStack cloud via a verifier."""
@@ -592,32 +594,53 @@ class VerifyCommands(object):
     @cliutils.help_group("verification")
     @cliutils.args("--uuid", nargs="+", dest="verification_uuid", type=str,
                    help="UUIDs of verifications. " + LIST_VERIFICATIONS_HINT)
-    @cliutils.args("--html", dest="html", action="store_true", required=False,
-                   help="Generate the report in HTML format instead of JSON.")
-    @cliutils.args("--file", dest="output_file", type=str,
-                   metavar="<path>", required=False,
-                   help="Path to a file to save the report to.")
+    @cliutils.args("--type", dest="output_type", type=str,
+                   required=False, default="json",
+                   help="Report type (Defaults to JSON). Out of the box types:"
+                        " %s. HINT: You can list all types by executing "
+                        "`rally plugins list` command."
+                        % ", ".join(DEFAULTS_REPORTERS))
+    @cliutils.args("--to", dest="output_dest", type=str,
+                   metavar="<dest>", required=False,
+                   help="Report destination. Can be a path to a file (in case"
+                        " of HTML, JSON types) to save the report to or "
+                        "a connection string. It depends on report type.")
     @cliutils.args("--open", dest="open_it", action="store_true",
                    required=False, help="Open the output file in a browser.")
     @envutils.with_default_verification_uuid
-    def report(self, api, verification_uuid=None, html=False, output_file=None,
-               open_it=False):
+    @plugins.ensure_plugins_are_loaded
+    def report(self, api, verification_uuid=None, output_type=None,
+               output_dest=None, open_it=None):
         """Generate a report for a verification or a few verifications."""
 
-        # TODO(ylobankov): Add support for CSV format.
-
         if not isinstance(verification_uuid, list):
             verification_uuid = [verification_uuid]
-        raw_report = api.verification.report(verification_uuid, html)
 
-        if output_file:
-            with open(output_file, "w") as f:
-                f.write(raw_report)
+        result = api.verification.report(verification_uuid, output_type,
+                                         output_dest)
+        if "files" in result:
+            print("Saving the report to disk. It can take a moment.")
+            for path in result["files"]:
+                full_path = os.path.abspath(os.path.expanduser(path))
+                if not os.path.exists(os.path.dirname(full_path)):
+                    os.makedirs(os.path.dirname(full_path))
+                with open(full_path, "w") as f:
+                    f.write(result["files"][path])
+                print(_("The report has been successfully saved."))
+
             if open_it:
+                if "open" not in result:
+                    print(_("Cannot open '%s' report in the browser because "
+                            "report type doesn't support it.") % output_type)
+                    return 1
                 webbrowser.open_new_tab(
-                    "file://" + os.path.realpath(output_file))
-        else:
-            print(raw_report)
+                    "file://" + os.path.abspath(result["open"]))
+
+        if "print" in result:
+            # NOTE(andreykurilin): we need a separation between logs and
+            #   printed information to be able parsing output
+            print(cliutils.make_header("Verification Report"))
+            print(result["print"])
 
     @cliutils.help_group("verification")
     @cliutils.args("--verifier-id", dest="verifier_id", type=str,

rally/plugins/common/verification/reporters.py

@@ -1,8 +1,5 @@
-# Copyright 2016: Mirantis Inc.
 # All Rights Reserved.
 #
-#    Author: Oleksandr Savatieiev osavatieiev@mirantis.com
-#
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
 #    a copy of the License at
@@ -16,25 +13,35 @@
 #    under the License.
 
 import collections
-import copy
 import json
 import re
 
 from rally.ui import utils
+from rally.verification import reporter
 
 
 SKIP_RE = re.compile("Skipped until Bug: ?(?P<bug_number>\d+) is resolved.")
 LP_BUG_LINK = "https://launchpad.net/bugs/%s"
 
 
-class VerificationReport(object):
-    """Generate a report for a verification or a few verifications."""
+@reporter.configure("json")
+class JSONReporter(reporter.VerificationReporter):
+    @classmethod
+    def validate(cls, output_destination):
+        """Validate destination of report.
+
+        :param output_destination: Destination of report
+        """
+        # nothing to check :)
+        pass
+
+    def _generate(self):
+        """Prepare raw report."""
 
-    def __init__(self, verifications_list):
         verifications = collections.OrderedDict()
         tests = {}
 
-        for v in verifications_list:
+        for v in self.verifications:
             verifications[v.uuid] = {
                 "started_at": v.created_at.strftime("%Y-%m-%d %H:%M:%S"),
                 "finished_at": v.updated_at.strftime("%Y-%m-%d %H:%M:%S"),
@@ -59,6 +66,11 @@ class VerificationReport(object):
                                       "name": result["name"],
                                       "by_verification": {}}
 
+                tests[test_id]["by_verification"][v.uuid] = {
+                    "status": result["status"],
+                    "duration": result["duration"]
+                }
+
                 reason = result.get("reason", "")
                 if reason:
                     match = SKIP_RE.match(reason)
@@ -68,19 +80,26 @@ class VerificationReport(object):
                                         reason)
                 traceback = result.get("traceback", "")
                 sep = "\n\n" if reason and traceback else ""
-                details = (reason + sep + traceback.strip()) or None
+                d = (reason + sep + traceback.strip()) or None
+                if d:
+                    tests[test_id]["by_verification"][v.uuid]["details"] = d
 
-                tests[test_id]["by_verification"][v.uuid] = {
-                    "status": result["status"],
-                    "duration": result["duration"],
-                    "details": details
-                }
+        return {"verifications": verifications, "tests": tests}
 
-        self.report = {"verifications": verifications, "tests": tests}
+    def generate(self):
+        raw_report = json.dumps(self._generate(), indent=4)
 
-    def to_html(self):
-        """Generate HTML report."""
-        report = copy.deepcopy(self.report)
+        if self.output_destination:
+            return {"files": {self.output_destination: raw_report},
+                    "open": self.output_destination}
+        else:
+            return {"print": raw_report}
+
+
+@reporter.configure("html")
+class HTMLReporter(JSONReporter):
+    def generate(self):
+        report = self._generate()
         uuids = report["verifications"].keys()
         show_comparison_note = False
 
@@ -89,9 +108,10 @@ class VerificationReport(object):
             # at JS side
             test["has_details"] = False
             for test_info in test["by_verification"].values():
-                if test_info["details"]:
+                if "details" not in test_info:
+                    test_info["details"] = None
+                elif not test["has_details"]:
                     test["has_details"] = True
-                    break
 
             durations = []
             # iter by uuids to store right order for comparison
@@ -125,8 +145,13 @@ class VerificationReport(object):
                    "tests": report["tests"],
                    "show_comparison_note": show_comparison_note}
 
-        return template.render(data=json.dumps(context), include_libs=False)
+        raw_report = template.render(data=json.dumps(context),
+                                     include_libs=False)
 
-    def to_json(self, indent=4):
-        """Generate JSON report."""
-        return json.dumps(self.report, indent=indent)
+        # in future we will support html_static and will need to save more
+        # files
+        if self.output_destination:
+            return {"files": {self.output_destination: raw_report},
+                    "open": self.output_destination}
+        else:
+            return {"print": raw_report}

rally/verification/exporter.py

@@ -1,43 +0,0 @@
-# Copyright 2016: Mirantis Inc.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-
-"""
-Exporter - its the mechanism for exporting rally tasks into some specified
-system by connection string.
-"""
-
-import abc
-
-import six
-
-from rally.common.plugin import plugin
-
-
-configure = plugin.configure
-
-
-@plugin.base()
-@six.add_metaclass(abc.ABCMeta)
-class VerifyExporter(plugin.Plugin):
-
-    @abc.abstractmethod
-    def export(self, verification_uuid, connection_string):
-        """Export results of the task to the task storage.
-
-        :param verification_uuid: uuid of verification results
-        :param connection_string: string used to connect
-                                  to the external system
-        """

rally/verification/reporter.py

@@ -0,0 +1,103 @@
+# Copyright 2016: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+"""
+Reporter - its the mechanism for exporting rally verification into specified
+system or formats.
+"""
+
+import abc
+
+import jsonschema
+import six
+
+from rally.common.plugin import plugin
+from rally import consts
+
+
+configure = plugin.configure
+
+
+REPORT_RESPONSE_SCHEMA = {
+    "type": "object",
+    "$schema": consts.JSON_SCHEMA,
+    "properties": {
+        "files": {
+            "type": "object",
+            "patternProperties": {
+                ".{1,}": {"type": "string"}
+            }
+        },
+        "open": {
+            "type": "string",
+        },
+        "print": {
+            "type": "string"
+        }
+    },
+    "additionalProperties": False
+}
+
+
+@plugin.base()
+@six.add_metaclass(abc.ABCMeta)
+class VerificationReporter(plugin.Plugin):
+    def __init__(self, verifications, output_destination):
+        """Init reporter
+
+        :param verifications: list of results to generate report for
+        :param output_destination: destination of report
+        """
+        super(VerificationReporter, self).__init__()
+        self.verifications = verifications
+        self.output_destination = output_destination
+
+    @classmethod
+    @abc.abstractmethod
+    def validate(cls, output_destination):
+        """Validate destination of report.
+
+        :param output_destination: Destination of report
+        """
+
+    @abc.abstractmethod
+    def generate(self):
+        """Generate report
+
+        :returns: a dict with 3 optional elements:
+            - key "files" with a dictionary of files to save on disk.
+              keys are paths, values are contents;
+            - key "print" - data to print at cli level
+            - key "open" - path to file which should be open in case of
+              --open flag
+        """
+
+    @staticmethod
+    def make(reporter_cls, verifications, output_destination):
+        """Initialize reporter, generate and validate report.
+
+        :param reporter_cls: class of VerificationReporter to be used
+        :param verifications: list of results to generate report for
+        :param output_destination: destination of report
+
+        NOTE(andreykurilin): this staticmethod is designed to ensure that
+         output of reporter in right format.
+        """
+        report = reporter_cls(verifications, output_destination).generate()
+
+        jsonschema.validate(report, REPORT_RESPONSE_SCHEMA)
+
+        return report

tests/ci/rally_verify.py

@@ -73,9 +73,8 @@ def call_rally(cmd, print_output=False, output_type=None):
     if output_type:
         data["output_file"] = data["stdout_file"].replace(
             ".txt.", ".%s." % output_type)
-        data["cmd"] += " --file %s" % data["output_file"]
-        if output_type == "html":
-            data["cmd"] += " --html"
+        data["cmd"] += " --to %s" % data["output_file"]
+        data["cmd"] += " --type %s" % output_type
 
     try:
         LOG.info("Try to execute `%s`." % data["cmd"])
@@ -113,8 +112,9 @@ def start_verification(args):
     results["uuid"] = envutils.get_global(envutils.ENV_VERIFICATION)
     results["show"] = call_rally("verify show")
     results["show_detailed"] = call_rally("verify show --detailed")
-    for ot in ("json", "html"):
-        results[ot] = call_rally("verify report", output_type=ot)
+    for output_type in ("json", "html"):
+        results[output_type] = call_rally("verify report",
+                                          output_type=output_type)
     # NOTE(andreykurilin): we need to clean verification uuid from global
     # environment to be able to load it next time(for another verification).
     envutils.clear_global(envutils.ENV_VERIFICATION)
@@ -132,9 +132,10 @@ def write_file(filename, data):
 def generate_trends_reports(uuid_1, uuid_2):
     """Generate trends reports."""
     results = {}
-    for ot in ("json", "html"):
-        results[ot] = call_rally(
-            "verify report --uuid %s %s" % (uuid_1, uuid_2), output_type=ot)
+    for output_type in ("json", "html"):
+        results[output_type] = call_rally(
+            "verify report --uuid %s %s" % (uuid_1, uuid_2),
+            output_type=output_type)
     return results
 
 

tests/unit/cli/commands/test_verify.py

@@ -22,6 +22,8 @@ import six
 from rally.cli import cliutils
 from rally.cli.commands import verify
 from rally.cli import envutils
+from rally import plugins
+from rally.verification import reporter
 from tests.unit import fakes
 from tests.unit import test
 
@@ -381,21 +383,39 @@ class VerifyCommandsTestCase(test.TestCase):
         self.fake_api.verification.delete.assert_has_calls(
             [mock.call("v1_uuid"), mock.call("v2_uuid")])
 
+    @mock.patch("rally.cli.commands.verify.os")
     @mock.patch("rally.cli.commands.verify.webbrowser.open_new_tab")
     @mock.patch("rally.cli.commands.verify.open", create=True)
-    def test_report(self, mock_open, mock_open_new_tab):
-        self.verify.report(self.fake_api, "v_uuid", html=True,
-                           output_file="/p/a/t/h", open_it=True)
+    def test_report(self, mock_open, mock_open_new_tab, mock_os):
+        output_dest = "/p/a/t/h"
+        output_type = "type"
+        content = "content"
+        self.fake_api.verification.report.return_value = {
+            "files": {output_dest: content}, "open": output_dest}
+        mock_os.path.exists.return_value = False
+
+        self.verify.report(self.fake_api, "v_uuid", output_type=output_type,
+                           output_dest=output_dest, open_it=True)
         self.fake_api.verification.report.assert_called_once_with(
-            ["v_uuid"], True)
-        mock_open.assert_called_once_with("/p/a/t/h", "w")
-        mock_open_new_tab.assert_called_once_with("file:///p/a/t/h")
+            ["v_uuid"], output_type, output_dest)
+        mock_open.assert_called_once_with(mock_os.path.abspath.return_value,
+                                          "w")
+        mock_os.makedirs.assert_called_once_with(
+            mock_os.path.dirname.return_value)
 
         mock_open.reset_mock()
         mock_open_new_tab.reset_mock()
-        self.verify.report(self.fake_api, "v_uuid")
-        self.assertFalse(mock_open.called)
+        mock_os.makedirs.reset_mock()
+
+        mock_os.path.exists.return_value = True
+        self.fake_api.verification.report.return_value = {
+            "files": {output_dest: content}, "print": "foo"}
+
+        self.verify.report(self.fake_api, "v_uuid", output_type=output_type,
+                           output_dest=output_dest)
+
         self.assertFalse(mock_open_new_tab.called)
+        self.assertFalse(mock_os.makedirs.called)
 
     @mock.patch("rally.cli.commands.verify.VerifyCommands.use")
     @mock.patch("rally.cli.commands.verify.open", create=True)
@@ -429,3 +449,19 @@ class VerifyCommandsTestCase(test.TestCase):
         mock_use.reset_mock()
         self.verify.import_results(self.fake_api, "v_id", "d_id", do_use=False)
         self.assertFalse(mock_use.called)
+
+    @plugins.ensure_plugins_are_loaded
+    def test_default_reporters(self):
+        available_reporters = {
+            cls.get_name().lower()
+            for cls in reporter.VerificationReporter.get_all()
+            # ignore possible external plugins
+            if cls.__module__.startswith("rally")}
+        listed_in_cli = {name.lower() for name in verify.DEFAULTS_REPORTERS}
+        not_listed = available_reporters - listed_in_cli
+
+        if not_listed:
+            self.fail("All default reporters should be listed in "
+                      "%s.DEFAULTS_REPORTERS (case of letters doesn't matter)."
+                      " Missed reporters: %s" % (verify.__name__,
+                                                 ", ".join(not_listed)))

tests/unit/plugins/common/verification/test_reporters.py

@@ -18,121 +18,132 @@ import datetime as dt
 import mock
 
 from rally.common import utils
-from rally.ui import report
+from rally.plugins.common.verification import reporters
 from tests.unit import test
 
 
-class VerificationReportTestCase(test.TestCase):
-    def get_verifications(self):
-        tests_1 = {
-            "some.test.TestCase.test_foo[id=iiiidddd;smoke]":
-                {"name": "some.test.TestCase.test_foo",
-                 "tags": ["smoke", "id"],
-                 "status": "success",
-                 "duration": "8"},
-            "some.test.TestCase.test_skipped":
-                {"name": "some.test.TestCase.test_skipped",
-                 "status": "skip",
-                 "reason": "Skipped until Bug: 666 is resolved.",
-                 "duration": "0"},
-            "some.test.TestCase.test_xfail":
-                {"name": "some.test.TestCase.test_xfail",
-                 "status": "xfail",
-                 "reason": "something",
-                 "traceback": "HEEELP",
-                 "duration": "3"}
-        }
-        tests_2 = {
-            "some.test.TestCase.test_foo[id=iiiidddd;smoke]":
-                {"name": "some.test.TestCase.test_foo",
-                 "tags": ["smoke", "id"],
-                 "status": "success",
-                 "duration": "8"},
-            "some.test.TestCase.test_failed":
-                {"name": "some.test.TestCase.test_failed",
-                 "status": "fail",
-                 "traceback": "HEEEEEEELP",
-                 "duration": "8"},
-            "some.test.TestCase.test_skipped":
-                {"name": "some.test.TestCase.test_skipped",
-                 "status": "skip",
-                 "reason": "Skipped until Bug: 666 is resolved.",
-                 "duration": "0"},
-            "some.test.TestCase.test_xfail":
-                {"name": "some.test.TestCase.test_xfail",
-                 "status": "xfail",
-                 "reason": "something",
-                 "traceback": "HEEELP",
-                 "duration": "4"}
-        }
-        tests_3 = {
-            "some.test.TestCase.test_foo[id=iiiidddd;smoke]":
-                {"name": "some.test.TestCase.test_foo",
-                 "tags": ["smoke", "id"],
-                 "status": "success",
-                 "duration": "8"},
-            "some.test.TestCase.test_failed":
-                {"name": "some.test.TestCase.test_failed",
-                 "status": "fail",
-                 "traceback": "HEEEEEEELP",
-                 "duration": "7"},
-            "some.test.TestCase.test_skipped":
-                {"name": "some.test.TestCase.test_skipped",
-                 "status": "skip",
-                 "reason": "Skipped until Bug: 666 is resolved.",
-                 "duration": "0"},
-            "some.test.TestCase.test_xfail":
-                {"name": "some.test.TestCase.test_xfail",
-                 "status": "xfail",
-                 "reason": "something",
-                 "traceback": "HEEELP",
-                 "duration": "3"}
-        }
+PATH = "rally.plugins.common.verification.reporters"
 
-        return [
-            utils.Struct(uuid="foo-bar-1",
-                         created_at=dt.datetime(2001, 1, 1),
-                         updated_at=dt.datetime(2001, 1, 2),
-                         status="finished",
-                         run_args="set_name=compute",
-                         tests_duration=1.111,
-                         tests_count=9,
-                         skipped=0,
-                         success=3,
-                         expected_failures=3,
-                         unexpected_success=2,
-                         failures=1,
-                         tests=tests_1),
-            utils.Struct(uuid="foo-bar-2",
-                         created_at=dt.datetime(2002, 1, 1),
-                         updated_at=dt.datetime(2002, 1, 2),
-                         status="finished",
-                         run_args="set_name=full",
-                         tests_duration=22.222,
-                         tests_count=99,
-                         skipped=0,
-                         success=33,
-                         expected_failures=33,
-                         unexpected_success=22,
-                         failures=11,
-                         tests=tests_2),
-            utils.Struct(uuid="foo-bar-3",
-                         created_at=dt.datetime(2003, 1, 1),
-                         updated_at=dt.datetime(2003, 1, 2),
-                         status="finished",
-                         run_args="set_name=full",
-                         tests_duration=33.333,
-                         tests_count=99,
-                         skipped=0,
-                         success=33,
-                         expected_failures=33,
-                         unexpected_success=22,
-                         failures=11,
-                         tests=tests_3)
-        ]
 
-    def test__init__(self):
-        vreport = report.VerificationReport(self.get_verifications())
+def get_verifications():
+    tests_1 = {
+        "some.test.TestCase.test_foo[id=iiiidddd;smoke]":
+            {"name": "some.test.TestCase.test_foo",
+             "tags": ["smoke", "id"],
+             "status": "success",
+             "duration": "8"},
+        "some.test.TestCase.test_skipped":
+            {"name": "some.test.TestCase.test_skipped",
+             "status": "skip",
+             "reason": "Skipped until Bug: 666 is resolved.",
+             "duration": "0"},
+        "some.test.TestCase.test_xfail":
+            {"name": "some.test.TestCase.test_xfail",
+             "status": "xfail",
+             "reason": "something",
+             "traceback": "HEEELP",
+             "duration": "3"}
+    }
+    tests_2 = {
+        "some.test.TestCase.test_foo[id=iiiidddd;smoke]":
+            {"name": "some.test.TestCase.test_foo",
+             "tags": ["smoke", "id"],
+             "status": "success",
+             "duration": "8"},
+        "some.test.TestCase.test_failed":
+            {"name": "some.test.TestCase.test_failed",
+             "status": "fail",
+             "traceback": "HEEEEEEELP",
+             "duration": "8"},
+        "some.test.TestCase.test_skipped":
+            {"name": "some.test.TestCase.test_skipped",
+             "status": "skip",
+             "reason": "Skipped until Bug: 666 is resolved.",
+             "duration": "0"},
+        "some.test.TestCase.test_xfail":
+            {"name": "some.test.TestCase.test_xfail",
+             "status": "xfail",
+             "reason": "something",
+             "traceback": "HEEELP",
+             "duration": "4"}
+    }
+    tests_3 = {
+        "some.test.TestCase.test_foo[id=iiiidddd;smoke]":
+            {"name": "some.test.TestCase.test_foo",
+             "tags": ["smoke", "id"],
+             "status": "success",
+             "duration": "8"},
+        "some.test.TestCase.test_failed":
+            {"name": "some.test.TestCase.test_failed",
+             "status": "fail",
+             "traceback": "HEEEEEEELP",
+             "duration": "7"},
+        "some.test.TestCase.test_skipped":
+            {"name": "some.test.TestCase.test_skipped",
+             "status": "skip",
+             "reason": "Skipped until Bug: 666 is resolved.",
+             "duration": "0"},
+        "some.test.TestCase.test_xfail":
+            {"name": "some.test.TestCase.test_xfail",
+             "status": "xfail",
+             "reason": "something",
+             "traceback": "HEEELP",
+             "duration": "3"}
+    }
+
+    return [
+        utils.Struct(uuid="foo-bar-1",
+                     created_at=dt.datetime(2001, 1, 1),
+                     updated_at=dt.datetime(2001, 1, 2),
+                     status="finished",
+                     run_args="set_name=compute",
+                     tests_duration=1.111,
+                     tests_count=9,
+                     skipped=0,
+                     success=3,
+                     expected_failures=3,
+                     unexpected_success=2,
+                     failures=1,
+                     tests=tests_1),
+        utils.Struct(uuid="foo-bar-2",
+                     created_at=dt.datetime(2002, 1, 1),
+                     updated_at=dt.datetime(2002, 1, 2),
+                     status="finished",
+                     run_args="set_name=full",
+                     tests_duration=22.222,
+                     tests_count=99,
+                     skipped=0,
+                     success=33,
+                     expected_failures=33,
+                     unexpected_success=22,
+                     failures=11,
+                     tests=tests_2),
+        utils.Struct(uuid="foo-bar-3",
+                     created_at=dt.datetime(2003, 1, 1),
+                     updated_at=dt.datetime(2003, 1, 2),
+                     status="finished",
+                     run_args="set_name=full",
+                     tests_duration=33.333,
+                     tests_count=99,
+                     skipped=0,
+                     success=33,
+                     expected_failures=33,
+                     unexpected_success=22,
+                     failures=11,
+                     tests=tests_3)
+    ]
+
+
+class JSONReporterTestCase(test.TestCase):
+    def test_validate(self):
+        # nothing should fail
+        reporters.JSONReporter.validate(mock.Mock())
+        reporters.JSONReporter.validate("")
+        reporters.JSONReporter.validate(None)
+
+    def test__generate(self):
+        reporter = reporters.JSONReporter(get_verifications(), None)
+        report = reporter._generate()
 
         self.assertEqual(
             collections.OrderedDict(
@@ -169,18 +180,15 @@ class VerificationReportTestCase(test.TestCase):
                                 "unexpected_success": 22,
                                 "expected_failures": 33,
                                 "failures": 11})]),
-            vreport.report["verifications"])
+            report["verifications"])
 
         self.assertEqual({
             "some.test.TestCase.test_foo[id=iiiidddd;smoke]": {
-                "by_verification": {"foo-bar-1": {"details": None,
-                                                  "duration": "8",
+                "by_verification": {"foo-bar-1": {"duration": "8",
                                                   "status": "success"},
-                                    "foo-bar-2": {"details": None,
-                                                  "duration": "8",
+                                    "foo-bar-2": {"duration": "8",
                                                   "status": "success"},
-                                    "foo-bar-3": {"details": None,
-                                                  "duration": "8",
+                                    "foo-bar-3": {"duration": "8",
                                                   "status": "success"}
                                     },
                 "name": "some.test.TestCase.test_foo",
@@ -226,16 +234,40 @@ class VerificationReportTestCase(test.TestCase):
                                   "status": "xfail"}},
                 "name": "some.test.TestCase.test_xfail",
                 "tags": []}},
-            vreport.report["tests"])
+            report["tests"])
 
-    @mock.patch("rally.ui.report.utils")
-    @mock.patch("rally.ui.report.json.dumps")
-    def test_to_html(self, mock_dumps, mock_utils):
+    @mock.patch("%s.json.dumps" % PATH)
+    @mock.patch("%s.JSONReporter._generate" % PATH)
+    def test_generate(self, mock__generate, mock_dumps):
+        reporter = reporters.JSONReporter([], output_destination=None)
+        self.assertEqual({"print": mock_dumps.return_value},
+                         reporter.generate())
+        mock__generate.assert_called_once_with()
+        mock_dumps.assert_called_once_with(mock__generate.return_value,
+                                           indent=4)
+
+        mock__generate.reset_mock()
+        mock_dumps.reset_mock()
+
+        path = "some_path"
+        reporter = reporters.JSONReporter([], output_destination=path)
+        self.assertEqual({"files": {path: mock_dumps.return_value},
+                          "open": path}, reporter.generate())
+        mock__generate.assert_called_once_with()
+        mock_dumps.assert_called_once_with(mock__generate.return_value,
+                                           indent=4)
+
+
+class HTMLReporterTestCase(test.TestCase):
+    @mock.patch("%s.utils" % PATH)
+    @mock.patch("%s.json.dumps" % PATH)
+    def test_generate(self, mock_dumps, mock_utils):
         mock_render = mock_utils.get_template.return_value.render
 
-        vreport = report.VerificationReport(self.get_verifications())
+        reporter = reporters.HTMLReporter(get_verifications(), None)
 
-        self.assertEqual(mock_render.return_value, vreport.to_html())
+        self.assertEqual({"print": mock_render.return_value},
+                         reporter.generate())
         mock_render.assert_called_once_with(data=mock_dumps.return_value,
                                             include_libs=False)
         mock_utils.get_template.assert_called_once_with(
@@ -251,8 +283,6 @@ class VerificationReportTestCase(test.TestCase):
                          set(ctx.keys()))
         self.assertEqual(["foo-bar-1", "foo-bar-2", "foo-bar-3"],
                          list(ctx["uuids"]))
-        self.assertEqual(vreport.report["verifications"],
-                         ctx["verifications"])
         self.assertTrue(ctx["show_comparison_note"])
         self.assertEqual({
             "some.test.TestCase.test_foo[id=iiiidddd;smoke]": {
@@ -310,15 +340,3 @@ class VerificationReportTestCase(test.TestCase):
                 "name": "some.test.TestCase.test_xfail",
                 "tags": []}},
             ctx["tests"])
-
-    @mock.patch("rally.ui.report.json.dumps")
-    def test_to_json(self, mock_dumps):
-        obj = mock.MagicMock()
-        indent = 777
-
-        vreport = report.VerificationReport([])
-        vreport.report = obj
-
-        self.assertEqual(mock_dumps.return_value, vreport.to_json(indent))
-
-        mock_dumps.assert_called_once_with(obj, indent=indent)

tests/unit/test_api.py

@@ -1130,31 +1130,27 @@ class VerificationAPITestCase(test.TestCase):
         mock_verification_list.assert_called_once_with(
             verifier_id, deployment_id=deployment_id, status=status)
 
-    @mock.patch("rally.api.report.VerificationReport")
+    @mock.patch("rally.api.vreporter.VerificationReporter")
     @mock.patch("rally.api.objects.Verification.get")
-    def test_report(self, mock_verification_get, mock_verification_report):
+    def test_report(self, mock_verification_get, mock_verification_reporter):
         verifications = ["uuid-1", "uuid-2"]
+        output_type = mock.Mock()
+        output_dest = mock.Mock()
 
-        vreport_obj = mock_verification_report.return_value
+        reporter = mock_verification_reporter.get.return_value
 
-        self.assertEqual(vreport_obj.to_html.return_value,
-                         api._Verification.report(verifications, html=True))
-        vreport_obj.to_html.assert_called_once_with()
-        mock_verification_report.assert_called_once_with(
-            [mock_verification_get.return_value,
-             mock_verification_get.return_value])
-        self.assertEqual([mock.call(u) for u in verifications],
-                         mock_verification_get.call_args_list)
+        self.assertEqual(mock_verification_reporter.make.return_value,
+                         api._Verification.report(verifications,
+                                                  output_type=output_type,
+                                                  output_dest=output_dest))
+        mock_verification_reporter.get.assert_called_once_with(output_type)
 
-        mock_verification_get.reset_mock()
-        mock_verification_report.reset_mock()
+        reporter.validate.assert_called_once_with(output_dest)
 
-        self.assertEqual(vreport_obj.to_json.return_value,
-                         api._Verification.report(verifications))
-        vreport_obj.to_json.assert_called_once_with()
-        mock_verification_report.assert_called_once_with(
-            [mock_verification_get.return_value,
-             mock_verification_get.return_value])
+        mock_verification_reporter.make.assert_called_once_with(
+            reporter, [mock_verification_get.return_value,
+                       mock_verification_get.return_value],
+            output_dest)
         self.assertEqual([mock.call(u) for u in verifications],
                          mock_verification_get.call_args_list)
 

tests/unit/verification/test_exporter.py

@@ -1,33 +0,0 @@
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-from rally.common.plugin import plugin
-from rally.verification import exporter
-from tests.unit import test
-
-
-@plugin.configure(name="test_verify_exporter")
-class TestExporter(exporter.VerifyExporter):
-
-    def export(self, uuid, connection_string):
-        pass
-
-
-class ExporterTestCase(test.TestCase):
-
-    def test_task_export(self):
-        self.assertRaises(TypeError, exporter.VerifyExporter)
-
-    def test_task_export_instantiate(self):
-        TestExporter()

tests/unit/verification/test_reporter.py

@@ -0,0 +1,75 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import jsonschema
+import mock
+
+from rally.verification import reporter
+from tests.unit import test
+
+
+class ReporterTestCase(test.TestCase):
+
+    def test_make(self):
+        reporter_cls = mock.Mock()
+
+        reporter_cls.return_value.generate.return_value = {}
+        reporter.VerificationReporter.make(reporter_cls, None, None)
+
+        reporter_cls.return_value.generate.return_value = {"files": {}}
+        reporter.VerificationReporter.make(reporter_cls, None, None)
+        reporter_cls.return_value.generate.return_value = {
+            "files": {"/path/foo": "content"}}
+        reporter.VerificationReporter.make(reporter_cls, None, None)
+
+        reporter_cls.return_value.generate.return_value = {"open": "/path/foo"}
+        reporter.VerificationReporter.make(reporter_cls, None, None)
+
+        reporter_cls.return_value.generate.return_value = {"print": "foo"}
+        reporter.VerificationReporter.make(reporter_cls, None, None)
+
+        reporter_cls.return_value.generate.return_value = {
+            "files": {"/path/foo": "content"}, "open": "/path/foo",
+            "print": "foo"}
+        reporter.VerificationReporter.make(reporter_cls, None, None)
+
+        reporter_cls.return_value.generate.return_value = {"files": []}
+        self.assertRaises(jsonschema.ValidationError,
+                          reporter.VerificationReporter.make,
+                          reporter_cls, None, None)
+
+        reporter_cls.return_value.generate.return_value = {"files": ""}
+        self.assertRaises(jsonschema.ValidationError,
+                          reporter.VerificationReporter.make,
+                          reporter_cls, None, None)
+
+        reporter_cls.return_value.generate.return_value = {"files": {"a": {}}}
+        self.assertRaises(jsonschema.ValidationError,
+                          reporter.VerificationReporter.make,
+                          reporter_cls, None, None)
+
+        reporter_cls.return_value.generate.return_value = {"open": []}
+        self.assertRaises(jsonschema.ValidationError,
+                          reporter.VerificationReporter.make,
+                          reporter_cls, None, None)
+
+        reporter_cls.return_value.generate.return_value = {"print": []}
+        self.assertRaises(jsonschema.ValidationError,
+                          reporter.VerificationReporter.make,
+                          reporter_cls, None, None)
+
+        reporter_cls.return_value.generate.return_value = {"additional": ""}
+        self.assertRaises(jsonschema.ValidationError,
+                          reporter.VerificationReporter.make,
+                          reporter_cls, None, None)