[docs] Describe verification component

Change-Id: I5c227e9e76c322ebccda3788bed886f1c4870651

doc/source/overview/glossary.rst

@@ -1,5 +1,7 @@
 :tocdepth: 1
 
+.. _glossary:
+
 ========
 Glossary
 ========
@@ -165,6 +167,8 @@ Rally can run different subunit-based testing tools against a target
 environment, for example `tempest
 <http://docs.openstack.org/developer/tempest/>`_ for OpenStack.
 
+.. _glossary-verification:
+
 Verification
 ------------
 

doc/source/verification/cli_reference.rst

@@ -0,0 +1,27 @@
+..
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+.. _rally-verify-cli-reference:
+
+======================
+Command Line Interface
+======================
+
+Cut down from Global :ref:`cli-reference`
+
+.. contents::
+  :depth: 2
+  :local:
+
+.. make_cli_reference::
+  :group: verify

doc/source/verification/howto/add_new_reporter.rst

@@ -0,0 +1,104 @@
+..
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+.. _howto-add-new-reporting-mechanism:
+
+=================================
+HowTo add new reporting mechanism
+=================================
+
+Reporting mechanism for verifications is pluggable. Custom plugins can be used
+for custom output formats or for exporting results to external systems.
+
+We hardly recommend to read :ref:`plugins` page to understand how do Rally
+Plugins work.
+
+.. contents::
+  :depth: 2
+  :local:
+
+Spec
+----
+
+All reporters should inherit
+``rally.verification.reporter.VerificationReporter`` and implement all
+abstract methods. Here you can find its interface:
+
+    .. autoclass:: rally.verification.reporter.VerificationReporter
+       :members:
+
+Example of custom JSON Reporter
+-------------------------------
+
+Basically, you need to implement only two methods "validate" and "generate".
+
+Method "validate" should check that destination of the report is right.
+Method "generate" should build a report or export results somewhere; actually,
+it is up to you what it should do but return format is strict, see :ref:`spec`
+section for what it can return.
+
+.. code-block:: python
+
+    import json
+
+    from rally.verification import reporter
+
+
+    @reporter.configure("summary-in-json")
+    class SummaryInJsonReporter(reporter.VerificationReporter):
+        """Store summary of verification(s) in JSON format"""
+
+        # ISO 8601
+        TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
+
+        @classmethod
+        def validate(cls, output_destination):
+            # we do not have any restrictions for destination, so nothing to
+            # check
+            pass
+
+        def generate(self):
+            report = {}
+
+            for v in self.verifications:
+                report[v.uuid] = {
+                    "started_at": v.created_at.strftime(self.TIME_FORMAT),
+                    "finished_at": v.updated_at.strftime(self.TIME_FORMAT),
+                    "status": v.status,
+                    "run_args": v.run_args,
+                    "tests_count": v.tests_count,
+                    "tests_duration": v.tests_duration,
+                    "skipped": v.skipped,
+                    "success": v.success,
+                    "expected_failures": v.expected_failures,
+                    "unexpected_success": v.unexpected_success,
+                    "failures": v.failures,
+                    # v.tests includes all information about launched tests,
+                    # but for simplification of this fake reporters, let's
+                    # save just names
+                    "launched_tests": [test["name"]
+                                       for test in v.tests.values()]
+                }
+
+            raw_report = json.dumps(report, indent=4)
+
+            if self.output_destination:
+                # In case of output_destination existence report will be saved
+                # to hard drive and there is nothing to print to stdout, so
+                # "print" key is not used
+                return {"files": {self.output_destination: raw_report},
+                        "open": self.output_destination}
+            else:
+                # it is something that will be print at CLI layer.
+                return {"print": raw_report}
+

doc/source/verification/howto/add_support_for_new_tool.rst

@@ -0,0 +1,117 @@
+..
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+.. _howto-add-support-for-new-tool:
+
+==============================
+HowTo add support for new tool
+==============================
+
+First of all, you should start from the reading of :ref:`plugins` page.
+After you learned basic things about Rally plugin mechanism, let's move to
+Verifier interface itself.
+
+.. contents::
+  :depth: 2
+  :local:
+
+Spec
+----
+
+All verifiers plugins should inherit
+``rally.verification.manager.VerifierManager`` and implement all abstract
+methods. Here you can find its interface:
+
+    .. autoclass:: rally.verification.manager.VerifierManager
+       :members:
+       :exclude-members: base_ref, check_system_wide, checkout, install_venv,
+         parse_results, validate
+
+
+Example of Fake Verifier Manager
+--------------------------------
+
+FakeTool is a tool which doesn't require configuration and installation.
+
+  .. code-block:: python
+
+       import random
+       import re
+
+       from rally.verification import manager
+
+
+       # Verification component expects that method "run" of verifier returns
+       # object. Class Result is a simple wrapper for two expected properties.
+       class Result(object):
+           def __init__(self, totals, tests):
+               self.totals = totals
+               self.tests = tests
+
+
+       @manager.configure("fake-tool", default_repo="https://example.com")
+       class FakeTool(manager.VerifierManager):
+           """Fake Tool \o/"""
+
+           TESTS = ["fake_tool.tests.bar.FatalityTestCase.test_one",
+                    "fake_tool.tests.bar.FatalityTestCase.test_two",
+                    "fake_tool.tests.bar.FatalityTestCase.test_three",
+                    "fake_tool.tests.bar.FatalityTestCase.test_four",
+                    "fake_tool.tests.foo.MegaTestCase.test_one",
+                    "fake_tool.tests.foo.MegaTestCase.test_two",
+                    "fake_tool.tests.foo.MegaTestCase.test_three",
+                    "fake_tool.tests.foo.MegaTestCase.test_four"]
+
+           # This fake verifier doesn't launch anything, just returns random
+           #  results, so let's override parent methods to avoid redundant
+           #  clonning repo, checking packages and so on.
+
+           def install(self):
+               pass
+
+           def uninstall(self, full=False):
+               pass
+
+           # Each tool, which supports configuration, has the own mechanism
+           # for that task. Writing unified method is impossible. That is why
+           # `VerificationManager` implements the case when the tool doesn't
+           # need (doesn't support) configuration at all. Such behaviour is
+           # ideal for FakeTool, since we do not need to change anything :)
+
+           # Let's implement method `run` to return random data.
+           def run(self, context):
+
+               totals = {"tests_count": len(self.TESTS),
+                         "tests_duration": 0,
+                         "failures": 0,
+                         "skipped": 0,
+                         "success": 0,
+                         "unexpected_success": 0,
+                         "expected_failures": 0}
+               tests = {}
+               for name in self.TESTS:
+                   duration = random.randint(0, 10000)/100.
+                   totals["tests_duration"] += duration
+                   test = {"name": name,
+                           "status": random.choice(["success", "fail"]),
+                           "duration": "%s" % duration}
+                   if test["status"] == "fail":
+                       test["traceback"] = "Ooooppps"
+                       totals["failures"] += 1
+                   else:
+                       totals["success"] += 1
+                   tests[name] = test
+               return Result(totals, tests=tests)
+
+           def list_tests(self, pattern=""):
+               return [name for name in self.TESTS if re.match(pattern, name)]

doc/source/verification/howto/index.rst

@@ -0,0 +1,12 @@
+
+=====
+HowTo
+=====
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    ./add*
+    migrate_from_old_design
+

doc/source/verification/howto/migrate_from_old_design.rst

@@ -0,0 +1,510 @@
+..
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+========================================================
+HowTo migrate from Verification component 0.7.0 to 0.8.0
+========================================================
+
+.. note:: This document describes migration process from 0.7.0 to 0.8.0 Rally
+    version. You can apply this instruction for migration to later versions,
+    but check all references and release notes before trying to do it.
+
+
+Verification Component was introduced long time ago even before the first Rally
+release. It started as a small helper thing but became a big powerful tool.
+Since it was not designed to all features that were implemented there later,
+it contained a lot of workarounds and hacks.
+
+New Verification Component, which we are happy to introduce, should fix all
+architecture issues and improve user-experience. Unfortunately, fixing all
+those obsolete architecture decisions could not be done in a
+backward-compatible way, or it would produce much more workarounds. That is why
+we decided to redesign the whole component in a clear way - remove old code and
+write a new one from scratch.
+
+Migration to New Verification Component should be simple and do not take too
+much time. You can find description of made changes below.
+
+.. contents::
+  :depth: 2
+  :local:
+
+Reports
+-------
+
+We completely reworked verification reports and merged comparison to main
+report. Now you can build one report for multiple number of verifications.
+
+For more details follow :ref:`verification-reports`
+
+Verification statuses
+---------------------
+
++------------+------------+---------------------------------------------------+
+| Old Status | New Status | Description                                       |
++============+============+===================================================+
+| init       | init       | Initial state. It appears instantly after calling |
+|            |            | ``rally verify start`` command before the actual  |
+|            |            | run of verifier's tool.                           |
++------------+------------+---------------------------------------------------+
+| running    |            | It was used right after checking status of        |
+|            |            | verifier. It is redundant in terms of new design. |
++------------+------------+---------------------------------------------------+
+| verifying  | running    | Identifies the process of execution tool.         |
++------------+------------+---------------------------------------------------+
+| finished   | finished   | Previously, finished state was used for           |
+|            |            | identification of just finished verification. By  |
+|            |            | "finished" meant that verification has any test   |
+|            |            | result. Now it means that verification was        |
+|            |            | executed and doesn't have failures, unexpected    |
+|            |            | success or any kind of errors.                    |
+|            +------------+---------------------------------------------------+
+|            | failed     | Old purpose is identification "errors", situations|
+|            |            | when results are empty. The right use is          |
+|            |            | an identification finished verification with      |
+|            |            | tests in failed and unexpected success statuses.  |
++------------+------------+---------------------------------------------------+
+| failed     | crashed    | Something went wrong while launching verification.|
++------------+------------+---------------------------------------------------+
+
+Latest information about verification statuses you can find at
+:ref:`verification_statuses`.
+
+Command Line Interface
+----------------------
+
+You can find the latest information about Verification Component CLI here -
+:ref:`rally-verify-cli-reference`.
+
+Installing verifier
+"""""""""""""""""""
+
+Command for Rally 0.7.0 - `rally verify install
+<http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-install>`_
+
+.. code-block:: console
+
+  $ rally verify install --deployment <uuid> --source <url> --version <vers> \
+    --system-wide
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify create-verifier --type "tempest" --source <url> \
+    --version <version> --system-wide
+
+Here you can find several important improvements:
+
+1) Rally team introduced new entity - :ref:`verifiers`. Verifier stores all
+   information about installed tool (i.e., source, version, system-wide) in a
+   database. You do not need to transmit the same arguments into
+   all ``rally verify`` commands as it was previously with ``--system-wide``
+   flag.
+
+2) You can use particular verifier for multiple deployments. ``--deployment``
+   flag moved to ``rally verify start`` command. Also, you can run it
+   simultaneously (checking in parallel different sets, different cloud, etc)
+
+3) Verification Component can use not only Tempest for verifying system. Check
+   :ref:`known-verifier-types` for full list of supported tools.
+
+4) You can have unlimited number of verifiers.
+
+Re-install verifier aka update
+""""""""""""""""""""""""""""""
+
+Command for Rally 0.7.0 - `rally verify reinstall
+<http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-reinstall>`_
+
+.. code-block:: console
+
+  $ rally verify reinstall --deployment <uuid> --source <url> --version <vers> \
+    --system-wide
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify update-verifier --id <id> --source <url> --version <vers> \
+    --system-wide --no-system-wide --update-venv
+
+Changes:
+
+1) ``rally verify update-verifier`` doesn't require deployment id
+
+2) You can switch between usage of system-wide installation and virtual
+   environment.
+
+3) You can update just virtual environment without cloning verifier code again
+
+Uninstall
+"""""""""
+
+Command for Rally 0.7.0 - `rally verify uninstall
+<http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-uninstall>`_
+
+.. code-block:: console
+
+  $ rally verify uninstall --deployment <uuid>
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify delete-verifier --id <id> --deployment-id <id> --force
+
+Changes:
+
+1) As it was mentioned before, Verifier doesn't have an alignment to any
+   particular deployment, so deployment argument is optional now.
+   If --deployment-id argument is specified only deployment specific data will
+   be removed (i.e, configurations).
+
+2) New --force flag for removing all verifications results for that verifier.
+
+Installation extensions
+"""""""""""""""""""""""
+
+Command for Rally 0.7.0 - `rally verify installplugin
+<http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-installplugin>`_
+
+.. code-block:: console
+
+  $ rally verify installplugin --deployment <uuid> --source <url> \
+    --version <vers> --system-wide
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify add-verifier-ext --id <id> --source <url> --version <vers> \
+    --extra-settings <data>
+
+Changes:
+
+1) --system-wide flag is removed. Rally checks the verifier information to
+   identify where to install the extension - in a system-side way or use
+   virtual environment.
+
+2) New --extra-settings flag. In case of Tempest, it is redundant, but for
+   other verifiers allows to transmit some extra installation settings for
+   verifier extension.
+
+Uninstall extensions
+""""""""""""""""""""
+
+Command for Rally 0.7.0 - `rally verify uninstallplugin
+<http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-uninstallplugin>`_
+
+
+.. code-block:: console
+
+  $ rally verify uninstallplugin --deployment <uuid> --repo-name <repo_name> \
+    --system-wide
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify delete-verifier-ext --id <id> --name <name>
+
+Changes:
+
+1) It is one more place where you do not need to pass --system-wide flag
+   anymore.
+
+2) --deployment flag is gone.
+
+3) --repo-name is renamed to just --name.
+
+List extensions
+"""""""""""""""
+
+Command for Rally 0.7.0 - `rally verify listplugins
+<http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-listplugins>`_
+
+.. code-block:: console
+
+  $ rally verify listplugins --deployment <uuid> --system-wide
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify list-verifier-exts --id <id>
+
+Changes:
+
+1) No need to specify --system-wide flag.
+
+2) --deployment flag is gone.
+
+Discover available tests
+""""""""""""""""""""""""
+
+Command for Rally 0.7.0 - `rally verify discover
+<http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-discover>`_
+
+.. code-block:: console
+
+  $ rally verify discover --deployment <uuid> --system-wide --pattern <pattern>
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify list-verifier-tests --id <id> --pattern <pattern>
+
+Changes:
+
+1) No need to specify --system-wide flag.
+
+2) --deployment flag is gone.
+
+Configuring
+"""""""""""
+
+Commands for Rally 0.7.0:
+
+* The command for generating configs `rally verify genconfig
+  <http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-genconfig>`_
+
+  .. code-block:: console
+
+    $ rally verify genconfig --deployment <uuid> --tempest-config <path> \
+      --add-options <path> --override
+
+* The command for showing configs `rally verify showconfig
+  <http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-showconfig>`_
+
+  .. code-block:: console
+
+    $ rally verify showconfig --deployment <uuid>
+
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify configure-verifier --id <id> --deployment-id <uuid> \
+    --extend <path/json/yaml> --override <path> --reconfigure --show
+
+Changes:
+
+1) The argument ``--override`` replaces old ``--tempest-config`` name. First
+   of all, argument name "override" is a unified word without alignment to any
+   tool. Also, it describes in the best way the meaning of the action: use
+   client specified configuration file.
+
+2) The argument ``--extend`` replaces old ``--add-options``. It accepts a path
+   to config in INI format or JSON/YAML string. In future, it will be extended
+   with the ability to specify a path to JSON/YAML file.
+
+3) The argument ``--reconfigure`` replaces old ``--override``. It means that
+   existing file will be ignored and new one will be used/created.
+
+4) If the argument ``--show`` is specified, a configuration of verifier will be
+   displayed at the end of command execution.
+
+   .. note:: We do not have a separate command for showing configurations
+     anymore. ``rally verify configure-verifier --show`` shows an existing
+     configuration if it exists and ``--reconfigure`` argument is not specified
+
+Show config
+"""""""""""
+
+Command for Rally 0.7.0 - `rally verify showconfig
+<http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-showconfig>`_
+
+.. code-block:: console
+
+  $ rally verify showconfig --deployment <uuid>
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify configure-verifier --id <id> --deployment-id <uuid> --show
+
+Changes:
+
+  We do not have a separate command for that task.
+  ``rally verify configure-verifier --show`` shows an existing configuration
+  (if it exists) in case of not specified ``--reconfigure`` argument.
+
+Running verification
+""""""""""""""""""""
+
+Command for Rally 0.7.0 - `rally verify start
+<http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-start>`_
+
+.. code-block:: console
+
+  $ rally verify start --deployment <uuid> --set <set_name> --regex <regex> \
+    --load-list <path> --tests-file <path> --skip-list <path> \
+    --tempest-config <path> --xfail-list <path> --system-wide \
+    --concurrency <N> --failing --no-use
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify start --id <id> --deployment-id <uuid> --pattern <pattern> \
+    --load-list <path> --skip-list <path> --xfail-list <path> \
+    --concurrency <N> --no-use
+
+Changes:
+
+1) You need to pass verifier id
+
+2) Arguments ``--set`` and ``--regex`` are merged in the new model to single
+   ``--pattern`` argument. Name of tests set should be specified like
+   ``--pattern set_name=<set_name>``. It was done to provide a way for each
+   verifier to support custom arguments.
+
+3) The argument ``--tests-file`` was deprecated in Rally 0.6.0 and
+   time&ability come to delete it.
+4) Arguments ``--skip-list`` and ``--xfail-list`` accept path to file in
+   JSON/YAML format. Content should be a dictionary, where keys are tests
+   names (full name with id and tags) and values are reasons.
+5) The argument ``--tempest-config`` is gone. Use
+   ``rally verify configure-verifier --id <id> --deployment-id <uuid>
+   --override <path>`` instead.
+6) The argument ``--system-wide`` is gone like in most of other commands.
+
+Show verification result
+""""""""""""""""""""""""
+
+Commands for Rally 0.7.0:
+
+* The command for showing results of verification `rally verify show
+  <http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-show>`_
+
+  .. code-block:: console
+
+    $ rally verify show --uuid <uuid> --sort-by <query> --detailed
+
+* Separate command witch call ``rally verify show`` with hardcoded
+  ``--detailed`` flag `rally verify showconfig
+  <http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-detailed>`_
+
+  .. code-block:: console
+
+    $ rally verify detailed --uuid <uuid> --sort-by <query>
+
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify show --uuid <uuid> --sort-by <query> --detailed
+
+Changes:
+
+1) Redundant ``rally verify detailed`` command is removed
+
+2) Sorting tests via ``--sort-by`` argument is extended to name/duration/status
+
+Listing all verifications
+"""""""""""""""""""""""""
+
+Command for Rally 0.7.0 - `rally verify list
+<http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-list>`_
+
+.. code-block:: console
+
+  $ rally verify list
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify list --id <id> --deployment-id <id> --status <status>
+
+Changes:
+
+  You can filter verifications by verifiers, by deployments and by results
+  statuses.
+
+Importing results
+"""""""""""""""""
+
+Command for Rally 0.7.0 - `rally verify import
+<http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-import>`_
+
+.. code-block:: console
+
+  $ rally verify import --deployment <uuid> --set <set_name> --file <path> --no-use
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify import --id <id> --deployment <uuid> --file <path> \
+    --run-args <run_args> --no-use
+
+Changes:
+
+1) You need to specify verifier to import results for.
+
+2) The argument ``--set`` is merged into unified ``--run-args``.
+
+Building reports
+""""""""""""""""
+
+Commands for Rally 0.7.0:
+
+* The command for building HTML/JSON reports of verification
+  `rally verify results
+  <http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-results>`_
+
+  .. code-block:: console
+
+    $ rally verify results --uuid <uuid> --html --json --output-file <path>
+
+* The command for comparison two verifications `rally verify compare
+  <http://rally.readthedocs.io/en/0.7.0/cli/cli_reference.html#rally-verify-compare>`_
+
+  .. code-block:: console
+
+    $ rally verify compare --uuid-1 <uuid_1> --uuid-2 <uuid_2> --csv --html \
+      --json --output-file <output_file> --threshold <threshold>
+
+Command for Rally 0.8.0:
+
+.. code-block:: console
+
+  $ rally verify report --uuid <uuid> --type <type> --to <destination> --open
+
+Changes:
+
+1) Building reports becomes pluggable. You can extend reporters types.
+   See :ref:`verification-reports` for more details.
+
+2) The argument ``--type`` expects type of report (HTML/JSON). There are no
+   more separate arguments for each report type.
+
+3) Reports are not aligned to only local types, so the argument ``--to``
+   replaces ``--output-file``. In case of HTML/JSON reports, it can include a
+   path to the local file like it was previously or URL to some external system
+   with credentials like ``https://username:password@example.com:777``.
+
+4) The comparison is embedded into main reports and it is now limited by two
+   verifications results. There are no reasons for the separate command for
+   that task.
+
+The End
+"""""""
+
+Have nice verifications!

doc/source/verification/index.rst

@@ -23,5 +23,12 @@ environment works by different tools (we call them
 :ref:`glossary-verification`).
 
 .. toctree::
-    :maxdepth: 3
+    :maxdepth: 2
     :glob:
+
+    verifiers
+    reports
+    cli_reference
+    howto/index
+
+.. include:: ./overview.rst

doc/source/verification/overview.rst

@@ -0,0 +1,57 @@
+..
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+Historical background
+---------------------
+
+Tempest, OpenStack’s official test suite, is a powerful tool for running a set
+of functional tests against an OpenStack cluster. Tempest automatically runs
+against every patch in every project of OpenStack, which lets us avoid merging
+changes that break functionality.
+
+Unfortunately, it has limited opportunities to be used, to process its results,
+etc. That is why we started Verification Component initiative a long time ago
+(see `a blog post
+<https://www.mirantis.com/blog/rally-openstack-tempest-testing-made-simpler/>`_
+for more details, but be careful as all user interface is changed completely
+since that time).
+
+What is Verification Component and why do you need it?
+------------------------------------------------------
+
+The primary goal of Rally Product is to provide a simple way to do complex
+things. As for functional testing, Verification component includes interfaces
+for:
+
+* **Managing things**. Create an isolated virtual environment and install
+  verification tool there? Yes, we can do it! Clone tool from Git repositories?
+  Sure! Store several versions of one tool (you know, sometimes they are
+  incompatible, with different required packages and so on)? Of course!
+  In general, Verification Component allows to install, upgrade, reinstall,
+  configure your tool. You should not care about zillion options anymore Rally
+  will discover them via cloud UX and make the configuration file for you
+  automatically.
+* **Launching verifiers**. Launchers of specific tools don't always contain all
+  required features, Rally team tries to fix this omission. Verification
+  Component support some of them like expected failures, the list of tests to
+  skip, a list of tests to launch, re-running previous verification or just
+  failed tests from it and so on. Btw, all verification runs arguments are
+  stored in the database.
+* **Processing results**. Rally DataBase stores all `verifications
+  <link-to-glossary>`_ and you can obtain unified (across different verifiers)
+  results at any time. You can find a verification run summary there, run
+  arguments which were used, error messages and etc. Comparison mechanism for
+  several verifications is available too. Verification reports can be generated
+  in several formats: HTML, JSON, JUnit-XML (see :ref:`verification-reports`
+  for more details). Also, reports mechanism is expendable and you can write
+  your own plugin for whatever system you want.

doc/source/verification/reports.rst

@@ -0,0 +1,114 @@
+..
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+.. _verification-reports:
+
+====================
+Verification reports
+====================
+
+Rally stores all verifications results in its DataBase so that you can access
+and process results at any time. No matter what verifier you use, results will
+be stored in a unified way and reports will be unified too.
+
+Out of the box, we support several types of reports out of the
+box: :include-var:`rally.cli.commands.verify.DEFAULT_REPORT_TYPES`; but our
+reporting system is pluggable so that you can write your own plugin to build
+some specific reports or to export results to the specific system (see
+:ref:`howto-add-new-reporting-mechanism` for more details`).
+
+.. contents::
+  :depth: 2
+  :local:
+
+HTML reports
+------------
+
+HTML report is the most convenient type of reports. It includes as much as
+possible useful information about Verifications.
+
+Here is an example of HTML report for 3 verifications.
+It was generated by next command:
+
+.. code-block:: console
+
+    $ rally verify report --uuid <uuid-1> <uuid-2> <uuid-3> --type html \
+      --to ./report.html
+
+
+.. image:: ../images/Report-Verify-for-4-Verifications.png
+   :align: center
+
+The report consists of two tables.
+
+First one is a summary table. It includes base information about
+verifications: UUIDs; numbers of tests; when they were launched; statuses; etc.
+Also, you can find detailed information grouped by tests statuses at the right
+part of the table.
+
+If the size (height) of the summary table seems too large for you and hinders
+to see more tests results, you can push "Toggle Header" button.
+
+The second table contains actual verifications results. They are grouped by
+tests names. The result of the test for particular verification overpainted by
+one of the next colours:
+
+* *Red* - It means that test has "failed" status
+* *Orange* - It is "unexpected success". Most of the parsers calculates it just
+  like failure
+* *Green* - Everything is ok. The test succeeded.
+* *Yellow* - It is "expected failure".
+* *Light Blue* - Test is skipped. It is nor good neither bad.
+
+Several verifications comparison is a default embedded behaviour of reports.
+The difference between verifications is displayed in brackets after actual
+test duration. Sign **+** means that current result is bigger that standard by
+the number going after the sign. Sign **-** is an opposite to **-**. Please,
+note that all diffs are comparisons with the first verification in a row.
+
+Filtering results
+"""""""""""""""""
+
+You can filter tests by setting or removing a mark from check box of the
+particular status column of the summary table.
+
+.. image:: ../images/Report-Verify-filter-by-status.png
+   :align: center
+
+Tests Tags
+""""""""""
+
+Some of the tests tools support tests tagging. It can be used for setting
+unique IDs, groups, etc. Usually, such tags are included in test name. It is
+inconvenient and Rally stores tags separately. By default they are hidden, but
+if you push "Toggle tags" button, they will be displayed under tests names.
+
+.. image:: ../images/Report-Verify-toggle-tags.png
+   :align: center
+
+Tracebacks & Reasons
+""""""""""""""""""""
+
+Tests with "failed" and "expected failure" statuses have tracebacks of
+failures. Tests with "skipped", "expected failure", "unexpected success" status
+has "reason" of events. By default, both tracebacks and reasons are hidden,
+but you can show them by clicking to the appropriate test.
+
+.. image:: ../images/Report-Verify-tracebacks.png
+   :align: center
+
+Plugins Reference for all out-of-the-box reporters
+--------------------------------------------------
+
+.. generate_plugin_reference::
+  :base_cls: Verification Reporter

doc/source/verification/verifiers.rst

@@ -0,0 +1,100 @@
+..
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+.. _verifiers:
+
+=========
+Verifiers
+=========
+
+.. contents::
+  :depth: 1
+  :local:
+
+What is it?
+-----------
+
+Verifier Plugin is a compatibility layer between Rally and the specific tool
+(such as Tempest) which runs tests. It implements features like installation,
+configuration, upgrades, running, etc in terms of the tool. It is a driver in
+other words.
+It is a pluggable entity, which means that you can easily add support for
+whatever tool you want (see :ref:`howto-add-support-for-new-tool` page for
+more information). Even more, you can deliver such plugin separately from Rally
+itself, but we firmly recommend to push a change to Rally upstream (see
+:ref:`contribute` guide), so Rally core-team will able to review it and help
+to improve.
+
+Verifier is an instance of the Verifier Plugin. It is an installed tool.
+For example, "Tempest" is a set of functional tests, it is Verifier Plugin
+(we have a plugin for it). Installed Tempest 12.0 from
+https://github.com/openstack/tempest in a virtual environment is the verifier.
+
+Verifier is not aligned to any particular deployment like it was in the past,
+you can use one verifier for testing unlimited number of deployments (each
+deployment will have separate configuration files for the tool).
+
+Verifier & Verifier Plugin are the main entities which Verification component
+operates with. Another one is the verifications results.
+
+Verifier statuses
+-----------------
+
+All verifiers can be in next statuses:
+
+* *init* - Initial state. It appears while you call ``rally verify
+  create-verifier`` command and installation step is not yet started.
+* *installing* - Installation of the verifier is not a quick task. It is about
+  cloning tool, checking packages or installing virtual environments with all
+  required packages. This state indicates that this step is in the process.
+* *installed* - It should be one of your favourite states. It means that
+  everything is ok and you can start verifying your cloud.
+* *updating* - This state identifies the process of updating verifier (version,
+  source, packages, etc.).
+* *extending* - The process of extending a verifier by its plugins.
+* *failed* - Something went wrong while installation.
+
+.. _verification_statuses:
+
+Verification statuses
+---------------------
+
+* *init* - Initial state. It appears instantly after calling
+  ``rally verify start`` command before the actual run of verifier's tool.
+* *running* - Identifies the process of execution tool.
+* *finished*- Verification is finished without errors and failures.
+* *failed* - Verification is finished, but there are some failed tests.
+* *crashed* - Unexpected error had happened while running verification.
+
+
+.. _known-verifier-types:
+
+Known verifier types
+--------------------
+
+Out of the box
+""""""""""""""
+
+You can execute command ``rally verify list-plugins`` locally to check
+available verifiers in your environment.
+
+Cut down from Global :ref:`plugin-reference` page:
+
+.. generate_plugin_reference::
+  :base_cls: Verifier Manager
+
+Third-party
+"""""""""""
+
+Nothing here yet.
+

rally/consts.py

@@ -202,6 +202,8 @@ class _VerifierStatus(utils.ImmutableMixin, utils.EnumMixin):
     FAILED = "failed"
 
 
+# NOTE(andreykurilin): In case of updating these statuses, please do not forget
+#   to update doc reference too
 class _VerificationStatus(utils.ImmutableMixin, utils.EnumMixin):
     """Verification statuses."""
     INIT = "init"

rally/plugins/openstack/verification/tempest/manager.py

@@ -30,13 +30,45 @@ from rally.verification import utils
 
 
 LOG = logging.getLogger(__name__)
+AVAILABLE_SETS = (list(consts.TempestTestSets) +
+                  list(consts.TempestApiTestSets) +
+                  list(consts.TempestScenarioTestSets))
 
 
 @manager.configure(name="tempest", namespace="openstack",
                    default_repo="https://git.openstack.org/openstack/tempest",
                    context={"tempest_configuration": {}, "testr_verifier": {}})
 class TempestManager(testr.TestrLauncher):
-    """Plugin for Tempest management."""
+    """Tempest verifier.
+
+    **Description**:
+
+    Quote from official documentation:
+
+      This is a set of integration tests to be run against a live OpenStack
+      cluster. Tempest has batteries of tests for OpenStack API validation,
+      Scenarios, and other specific tests useful in validating an OpenStack
+      deployment.
+
+    Rally supports features listed below:
+
+      * *cloning Tempest*: repository and version can be specified
+      * *installation*: system-wide with checking existence of required
+        packages or in virtual environment
+      * *configuration*: options are discovered via OpenStack API, but you can
+        override them if you need
+      * *running*: pre-creating all required resources(i.e images, tenants,
+        etc), prepare arguments, launching Tempest, live-progress output
+      * *results*: all verifications are stored in db, you can built reports,
+        compare verification at whatever you want time.
+
+    Appeared in Rally 0.8.0 *(actually, it appeared long time ago with first
+    revision of Verification Component, but 0.8.0 is mentioned since it is
+    first release after Verification Component redesign)*
+    """
+
+    RUN_ARGS = {"set_name": "Name of predefined sets of tests. Known names: "
+                            "%s" % ", ".join(AVAILABLE_SETS)}
 
     @property
     def run_environ(self):
@@ -158,14 +190,11 @@ class TempestManager(testr.TestrLauncher):
             if len(pattern) == 1:
                 pass  # it is just a regex
             elif pattern[0] == "set":
-                available_sets = (list(consts.TempestTestSets) +
+                if pattern[1] not in AVAILABLE_SETS:
-                                  list(consts.TempestApiTestSets) +
-                                  list(consts.TempestScenarioTestSets))
-                if pattern[1] not in available_sets:
                     raise exceptions.ValidationError(
                         "Test set '%s' not found in available "
                         "Tempest test sets. Available sets are '%s'."
-                        % (pattern[1], "', '".join(available_sets)))
+                        % (pattern[1], "', '".join(AVAILABLE_SETS)))
             else:
                 raise exceptions.ValidationError(
                     "'pattern' argument should be a regexp or set name "