[docs] Several improvements for extensions

* plugin_reference is extended to build data for particular Plugin Base cls
* plugin_reference is extended to build data for all Plugin Bases (instead
  of just building data for hardcoded classes).
* cli_reference is extended to build data for particular command
* new include_vars extension is introduced for including const data in docs

Change-Id: I76d9f85e41c6ab8e622f6d3581478962144525ba

doc/ext/cli_reference.py

@@ -188,11 +188,16 @@ def make_category_section(name, parser):
 
 
 class CLIReferenceDirective(rst.Directive):
+    optional_arguments = 1
+    option_spec = {"group": str}
 
     def run(self):
         parser = Parser()
         categories = copy.copy(main.categories)
         categories["db"] = manage.DBCommands
+        if "group" in self.options:
+            categories = {k: v for k,v in categories.items()
+                          if k == self.options["group"]}
         cliutils._add_command_parsers(categories, parser)
 
         content = []

doc/ext/include_vars.py

@@ -0,0 +1,58 @@
+# Copyright 2017: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from docutils import nodes
+import json
+
+from oslo_utils import importutils
+
+
+def include_var(name, rawtext, text, lineno, inliner, options=None,
+                content=None):
+    """
+
+
+    :param name: The local name of the interpreted role, the role name
+        actually used in the document.
+    :param rawtext: A string containing the enitre interpreted text input,
+        including the role and markup. Return it as a problematic node
+        linked to a system message if a problem is encountered.
+    :param text: The interpreted text content.
+    :param lineno: The line number where the interpreted text begins.
+    :param inliner: The docutils.parsers.rst.states.Inliner object that
+        called include_var. It contains the several attributes useful for
+        error reporting and document tree access.
+    :param options: A dictionary of directive options for customization
+        (from the "role" directive), to be interpreted by the role function.
+        Used for additional attributes for the generated elements and other
+        functionality.
+    :param content: A list of strings, the directive content for
+        customization (from the "role" directive). To be interpreted by the
+        role function.
+
+    :return:
+    """
+    obj = importutils.import_class(text)
+    if isinstance(obj, (tuple, list)):
+        obj = ", ".join(obj)
+    elif isinstance(obj, dict):
+        obj = json.dumps(dict, indent=4)
+    else:
+        obj = str(obj)
+    return [nodes.Text(obj)], []
+
+
+def setup(app):
+    app.add_role("include-var", include_var)

doc/ext/plugin_reference.py

@@ -14,124 +14,137 @@
 #    under the License.
 
 from docutils.parsers import rst
+import re
 
-from oslo_utils import importutils
+from rally.common.plugin import discover
-
+from rally.common.plugin import plugin
 from rally import plugins
-from utils import category, subcategory, paragraph, parse_text
+from utils import category, subcategory, section, paragraph, parse_text
-
-DATA = [
-    {
-        "group": "task",
-        "items": [
-            {
-                "name": "scenario runner",
-                "base": "rally.task.runner:ScenarioRunner"
-            },
-            {
-                "name": "SLA",
-                "base": "rally.task.sla:SLA"
-            },
-            {
-                "name": "context",
-                "base": "rally.task.context:Context"
-            },
-            {
-                "name": "scenario",
-                "base": "rally.task.scenario:Scenario"
-            }
-        ]
-    },
-    {
-        "group": "processing",
-        "items": [
-            {
-                "name": "output chart",
-                "base": "rally.task.processing.charts:OutputChart"
-            }
-        ]
-    },
-    {
-        "group": "deployment",
-        "items": [
-            {
-                "name": "engine",
-                "base": "rally.deployment.engine:Engine"
-            },
-            {
-                "name": "server provider",
-                "base":
-                "rally.deployment.serverprovider.provider:ProviderFactory"
-            }
-        ]
-    }
-]
 
 
-def _make_pretty_parameters(parameters):
+CATEGORIES = {
-    if not parameters:
+    "Common": ["OS Client"],
-        return ""
+    "Deployment": ["Engine", "Provider Factory"],
-
+    "Task Component": ["Chart", "Context", "Exporter", "Hook",
-    result = "**Parameters**:\n\n"
+                       "Resource Type", "SLA", "Scenario", "Scenario Runner",
-    for p in parameters:
+                       "Trigger"],
-        result += "* %(name)s: %(doc)s\n" % p
+    "Verification Component": ["Verifier Context", "Verification Reporter",
-    return result
+                               "Verifier Manager"]
+}
+# NOTE(andreykurilin): several bases do not have docstings at all, so it is
+# redundant to display them
+IGNORED_BASES = ["Resource Type", "Task Exporter", "OS Client"]
 
 
-def make_plugin_section(plugin, base_name):
+class PluginsReferenceDirective(rst.Directive):
-    subcategory_obj = subcategory("%s [%s]" % (plugin.get_name(), base_name))
+    optional_arguments = 1
-    info = plugin.get_info()
+    option_spec = {"base_cls": str}
-    if info["title"]:
-        subcategory_obj.append(paragraph(info["title"]))
 
-    if info["description"]:
+    @staticmethod
-        subcategory_obj.extend(parse_text(info["description"]))
+    def _make_pretty_parameters(parameters):
+        if not parameters:
+            return ""
 
-    if info["namespace"]:
+        result = "**Parameters**:\n\n"
-        subcategory_obj.append(paragraph(
+        for p in parameters:
+            result += "* %(name)s: %(doc)s\n" % p
+        return result
+
+    def _make_plugin_section(self, plugin_cls, base_name=None):
+        section_name = plugin_cls.get_name()
+        if base_name:
+            section_name += " [%s]" % base_name
+        section_obj = section(section_name)
+
+        info = plugin_cls.get_info()
+        if info["title"]:
+            section_obj.append(paragraph(info["title"]))
+
+        if info["description"]:
+            section_obj.extend(parse_text(info["description"]))
+
+        if info["namespace"]:
+            section_obj.append(paragraph(
                 "**Namespace**: %s" % info["namespace"]))
 
-    if info["parameters"]:
+        if info["parameters"]:
-        subcategory_obj.extend(parse_text(
+            section_obj.extend(parse_text(
-                _make_pretty_parameters(info["parameters"])))
+                self._make_pretty_parameters(info["parameters"])))
-        if info["returns"]:
+            if info["returns"]:
-            subcategory_obj.extend(parse_text(
+                section_obj.extend(parse_text(
                     "**Returns**:\n%s" % info["returns"]))
-    filename = info["module"].replace(".", "/")
+        filename = info["module"].replace(".", "/")
-    ref = "https://github.com/openstack/rally/blob/master/%s.py" % filename
+        ref = "https://github.com/openstack/rally/blob/master/%s.py" % filename
-    subcategory_obj.extend(parse_text("**Module**:\n`%s`__\n\n__ %s"
+        section_obj.extend(parse_text("**Module**:\n`%s`__\n\n__ %s"
-                                      % (info["module"], ref)))
+                                          % (info["module"], ref)))
-    return subcategory_obj
+        return section_obj
 
+    def _make_plugin_base_section(self, base_cls, base_name=None):
+        if base_name:
+            title = ("%ss" % base_name if base_name[-1] != "y"
+                     else "%sies" % base_name[:-1])
+            subcategory_obj = subcategory(title)
+        else:
+            subcategory_obj = []
+        for p in sorted(base_cls.get_all(), key=lambda o: o.get_name()):
+            subcategory_obj.append(self._make_plugin_section(p, base_name))
 
-def make_plugin_base_section(plugin_group):
+        return subcategory_obj
-    elements = []
 
-    for item in plugin_group["items"]:
+    @staticmethod
-        name = item["name"].title() if "SLA" != item["name"] else item["name"]
+    def _parse_class_name(cls):
-        category_obj = category("%s %ss" % (plugin_group["group"].title(),
+        name = ""
-                                            name))
+        for word in re.split(r'([A-Z][a-z]*)', cls.__name__):
-        elements.append(category_obj)
+            if word:
+                if len(word) > 1 and name:
+                    name += " "
+                name += word
+        return name
 
-        module, cls = item["base"].split(":")
+    def _get_all_plugins_bases(self):
-        plugin_base = getattr(importutils.import_module(module), cls)
+        """Return grouped and sorted all plugins bases."""
+        bases = []
+        bases_names = []
+        for p in discover.itersubclasses(plugin.Plugin):
+            base_ref = getattr(p, "base_ref", None)
+            if base_ref == p:
+                name = self._parse_class_name(p)
+                if name in bases_names:
+                    raise Exception("Two base classes with same name '%s' are "
+                                    "detected." % name)
+                bases_names.append(name)
+                category_of_base = "Common"
+                for cname, cbases in CATEGORIES.items():
+                    if name in cbases:
+                        category_of_base = cname
 
-        for p in plugin_base.get_all():
+                bases.append((category_of_base, name, p))
-            category_obj.append(make_plugin_section(p, item["name"]))
+        return sorted(bases)
-
-    return elements
-
-
-class PluginReferenceDirective(rst.Directive):
 
     def run(self):
-        content = []
+        plugins.load()
-        for i in range(len(DATA)):
+        bases = self._get_all_plugins_bases()
-            content.extend(make_plugin_base_section(DATA[i]))
+        if "base_cls" in self.options:
+            for _category_name, base_name, base_cls in bases:
+                if base_name == self.options["base_cls"]:
+                    return self._make_plugin_base_section(base_cls)
+            raise Exception("Failed to generate plugins reference for '%s'"
+                            " plugin base." % self.options["base_cls"])
 
-        return content
+        categories = {}
+
+        for category_name, base_name, base_cls in bases:
+            # FIXME(andreykurilin): do not ignore anything
+            if base_name in IGNORED_BASES:
+                continue
+            if category_name not in categories:
+                categories[category_name] = category(category_name)
+            category_of_base = categories[category_name]
+            category_of_base.append(self._make_plugin_base_section(base_cls,
+                                                                   base_name))
+        return [content for _name, content in sorted(categories.items())]
 
 
 def setup(app):
     plugins.load()
-    app.add_directive("generate_plugin_reference", PluginReferenceDirective)
+    app.add_directive("generate_plugin_reference", PluginsReferenceDirective)

doc/ext/utils.py

@@ -37,6 +37,7 @@ hint = lambda msg: nodes.hint("", *parse_text(msg))
 warning = lambda msg: nodes.warning("", paragraph(msg))
 category = lambda title: parse_text("%s\n%s" % (title, "-" * len(title)))[0]
 subcategory = lambda title: parse_text("%s\n%s" % (title, "~" * len(title)))[0]
+section = lambda title: parse_text("%s\n%s" % (title, "\"" * len(title)))[0]
 
 
 def make_definition(term, ref, descriptions):

doc/source/conf.py

@@ -60,7 +60,8 @@ extensions = [
     "sphinx.ext.ifconfig",
     "sphinx.ext.viewcode",
     "ext.cli_reference",
-    "ext.plugin_reference"
+    "ext.plugin_reference",
+    "ext.include_vars"
 ]
 todo_include_todos = True
 

doc/source/plugins/index.rst

@@ -18,16 +18,17 @@
 Rally Plugins
 =============
 
-
-Rally Plugin Reference
-----------------------
-
 Rally has a plugin oriented architecture - in other words Rally team is trying
 to make all places of code pluggable. Such architecture leads to the big amount
-of plugins. :ref:`Rally Plugins Reference page <plugin_reference>` contains
+of plugins. :ref:`plugin-reference` contains a full list of all official Rally
-a full list with detailed descriptions of all official Rally plugins.
+plugins with detailed descriptions.
 
 
+.. toctree::
+    :maxdepth: 1
+
+    plugin_reference
+
 How plugins work
 ----------------
 

doc/source/plugins/plugin_reference.rst

@@ -1,12 +1,13 @@
 :tocdepth: 1
-.. _plugin_reference:
+
+.. _plugin-reference:
 
 
-Rally Plugins Reference
+Plugins Reference
-=======================
+=================
 
 .. contents::
-  :depth: 1
+  :depth: 2
   :local:
 
 .. generate_plugin_reference::