Create api-ref docs site

This creates a stub for landing content to support having the source
for the api-ref site in our source tree. It includes a new sphinx
extension that will be used to generate the html needed for the site
which supports 2 new stanzas:

 .. rest_parameters::

This uses a parameters yaml file to look up the definitions of
parameters and produce a structured table of those parameters and how
they are used.

 .. rest_method::

This is a stanza used inside a section which specifies the REST method
(i.e. GET /servers/{id}) which is then used to produce a nice
collapsable section.

This uses quite a bit of the sphinx extension API, and tries to
document any non obvious actions.

This extension needs additional styling from bootstrap, and some
custom js / css to do the theming / collapsing. That's included as
part of this.

Change-Id: I41b568b782d3c85f6ef8d3bb3a6f4ae378e4dc33

api-ref/ext/rest_parameters.py

@@ -0,0 +1,319 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""This provides a sphinx extension able to create the HTML needed
+for the api-ref website.
+
+It contains 2 new stanzas.
+
+  .. rest_method:: GET /foo/bar
+
+Which is designed to be used as the first stanza in a new section to
+state that section is about that REST method. During processing the
+rest stanza will be reparented to be before the section in question,
+and used as a show/hide selector for it's details.
+
+  .. rest_parameters:: file.yaml
+
+     - name1: name_in_file1
+     - name2: name_in_file2
+     - name3: name_in_file3
+
+Which is designed to build structured tables for either response or
+request parameters. The stanza takes a value which is a file to lookup
+details about the parameters in question.
+
+The contents of the stanza are a yaml list of key / value pairs. The
+key is the name of the parameter to be shown in the table. The value
+is the key in the file.yaml where all other metadata about the
+parameter will be extracted. This allows for reusing parameter
+definitions widely in API definitions, but still providing for control
+in both naming and ordering of parameters at every declaration.
+
+"""
+
+from docutils import nodes
+from docutils.parsers.rst.directives.tables import Table
+from docutils.statemachine import ViewList
+from sphinx.util.compat import Directive
+
+import yaml
+
+
+def full_name(cls):
+    return cls.__module__ + '.' + cls.__name__
+
+
+class rest_method(nodes.Part, nodes.Element):
+    """rest_method custom node type
+
+    We specify a custom node type for rest_method so that we can
+    accumulate all the data about the rest method, but not render as
+    part of the normal rendering process. This means that we need a
+    renderer for every format we wish to support with this.
+
+    """
+    pass
+
+
+class RestMethodDirective(Directive):
+
+    # this enables content in the directive
+    has_content = True
+
+    def run(self):
+        lineno = self.state_machine.abs_line_number()
+        target = nodes.target()
+        section = nodes.section(classes=["detail-control"])
+        # env = self.state.document.settings.env
+        # env.app.info("Parent %s" % self.state.parent.attributes)
+
+        node = rest_method()
+
+        # TODO(sdague): this is a super simplistic parser, should be
+        # more robust.
+        method, sep, url = self.content[0].partition(' ')
+
+        node['method'] = method
+        node['url'] = url
+        node['target'] = self.state.parent.attributes['ids'][0]
+
+        # We need to build a temporary target that we can replace
+        # later in the processing to get the TOC to resolve correctly.
+        temp_target = "%s-selector" % node['target']
+        target = nodes.target(ids=[temp_target])
+        self.state.add_target(temp_target, '', target, lineno)
+        section += node
+
+        return [target, section]
+
+
+class RestParametersDirective(Table):
+
+    headers = ["Name", "In", "Type", "Description"]
+
+    def yaml_from_file(self, fpath):
+        """Collect Parameter stanzas from inline + file.
+
+        This allows use to reference an external file for the actual
+        parameter definitions.
+        """
+        # self.app.info("Fpath: %s" % fpath)
+        with open(fpath, 'r') as stream:
+            try:
+                lookup = yaml.load(stream)
+            except yaml.YAMLError as exc:
+                self.app.warn(exc)
+                raise
+
+        content = "\n".join(self.content)
+        parsed = yaml.load(content)
+        # self.app.info("Params loaded is %s" % parsed)
+        # self.app.info("Lookup table looks like %s" % lookup)
+        new_content = list()
+        for paramlist in parsed:
+            for name, ref in paramlist.items():
+                new_content.append((name, lookup[ref]))
+
+        # self.app.info("New content %s" % new_content)
+        self.yaml = new_content
+
+    def run(self):
+        env = self.state.document.settings.env
+        self.app = env.app
+
+        # Make sure we have some content, which should be yaml that
+        # defines some parameters.
+        if not self.content:
+            error = self.state_machine.reporter.error(
+                'No parameters defined',
+                nodes.literal_block(self.block_text, self.block_text),
+                line=self.lineno)
+            return [error]
+
+        if not len(self.arguments) >= 1:
+            self.state_machine.reporter.error(
+                'No reference file defined',
+                nodes.literal_block(self.block_text, self.block_text),
+                line=self.lineno)
+            return [error]
+
+        # NOTE(sdague): it's important that we pop the arg otherwise
+        # we end up putting the filename as the table caption.
+        rel_fpath, fpath = env.relfn2path(self.arguments.pop())
+        self.yaml_from_file(fpath)
+
+        self.max_cols = len(self.headers)
+        # TODO(sdague): it would be good to dynamically set column
+        # widths (or basically make the colwidth thing go away
+        # entirely)
+        self.options['widths'] = (20, 10, 10, 60)
+        self.col_widths = self.get_column_widths(self.max_cols)
+        # Actually convert the yaml
+        title, messages = self.make_title()
+        # self.app.info("Title %s, messages %s" % (title, messages))
+        table_node = self.build_table()
+        self.add_name(table_node)
+        if title:
+            table_node.insert(0, title)
+        return [table_node] + messages
+
+    def get_rows(self, table_data):
+        rows = []
+        groups = []
+        trow = nodes.row()
+        entry = nodes.entry()
+        para = nodes.paragraph(text=unicode(table_data))
+        entry += para
+        trow += entry
+        rows.append(trow)
+        return rows, groups
+
+    def collect_rows(self):
+
+        # Add a column for a field. In order to have the RST inside
+        # these fields get rendered, we need to use the
+        # ViewList. Note, ViewList expects a list of lines, so chunk
+        # up our content as a list to make it happy.
+        def add_col(value):
+            entry = nodes.entry()
+            result = ViewList(value.split('\n'))
+            self.state.nested_parse(result, 0, entry)
+            return entry
+
+        rows = []
+        groups = []
+        try:
+            # self.app.info("Parsed content is: %s" % self.yaml)
+            for key, values in self.yaml:
+                min_version = values.get('min_version', '')
+                desc = values.get('description', '')
+                classes = []
+                if min_version:
+                    desc += ("\n\n**New in version %s**\n" % min_version)
+                    min_ver_css_name = ("rp_min_ver_" +
+                                        str(min_version).replace('.', '_'))
+                    classes.append(min_ver_css_name)
+                trow = nodes.row(classes=classes)
+                name = key
+                if values.get('optional') is True:
+                    name += " (Optional)"
+                trow += add_col(name)
+                trow += add_col(values.get('in'))
+                trow += add_col(values.get('type'))
+                trow += add_col(desc)
+                rows.append(trow)
+        except AttributeError as exc:
+            self.app.warn("Failure on key: %s, values: %s. %s" %
+                          (key, values, exc))
+            raise
+        return rows, groups
+
+    def build_table(self):
+        table = nodes.table()
+        tgroup = nodes.tgroup(cols=len(self.headers))
+        table += tgroup
+
+        # TODO(sdague): it would be really nice to figure out how not
+        # to have this stanza, it kind of messes up all of the table
+        # formatting because it doesn't let tables just be the right
+        # size.
+        tgroup.extend(
+             nodes.colspec(colwidth=col_width, colname='c' + str(idx))
+             for idx, col_width in enumerate(self.col_widths)
+        )
+
+        thead = nodes.thead()
+        tgroup += thead
+
+        row_node = nodes.row()
+        thead += row_node
+        row_node.extend(nodes.entry(h, nodes.paragraph(text=h))
+                        for h in self.headers)
+
+        tbody = nodes.tbody()
+        tgroup += tbody
+
+        rows, groups = self.collect_rows()
+        tbody.extend(rows)
+        table.extend(groups)
+
+        return table
+
+
+def rest_method_html(self, node):
+    tmpl = """
+<div class="row operation-grp">
+    <div class="col-md-1 operation">
+    <a name="%(target)s" class="operation-anchor" href="#%(target)s">
+      <span class="glyphicon glyphicon-link"></span></a>
+    <span class="label label-success">%(method)s</span>
+    </div>
+    <div class="col-md-5">%(url)s</div>
+    <div class="col-md-5">%(desc)s</div>
+    <div class="col-md-1">
+    <button
+       class="btn btn-info btn-sm btn-detail"
+       data-target="#%(target)s-detail"
+       data-toggle="collapse"
+       id="%(target)s-detail-btn"
+       >detail</button>
+    </div>
+</div>"""
+
+    self.body.append(tmpl % node)
+    raise nodes.SkipNode
+
+
+def resolve_rest_references(app, doctree):
+    for node in doctree.traverse():
+        if isinstance(node, rest_method):
+            rest_node = node
+            rest_method_section = node.parent
+            rest_section = rest_method_section.parent
+            gp = rest_section.parent
+
+            # Added required classes to the top section
+            rest_section.attributes['classes'].append('api-detail')
+            rest_section.attributes['classes'].append('collapse')
+
+            # Pop the title off the collapsed section
+            title = rest_section.children.pop(0)
+            rest_node['desc'] = title.children[0]
+
+            # In order to get the links in the sidebar to be right, we
+            # have to do some id flipping here late in the game. The
+            # rest_method_section has basically had a dummy id up
+            # until this point just to keep it from colliding with
+            # it's parent.
+            rest_section.attributes['ids'][0] = (
+                "%s-detail" % rest_section.attributes['ids'][0])
+            rest_method_section.attributes['ids'][0] = rest_node['target']
+
+            # Pop the overall section into it's grand parent,
+            # right before where the current parent lives
+            idx = gp.children.index(rest_section)
+            rest_section.remove(rest_method_section)
+            gp.insert(idx, rest_method_section)
+
+
+def setup(app):
+    app.add_node(rest_method,
+                 html=(rest_method_html, None))
+    app.add_directive('rest_parameters', RestParametersDirective)
+    app.add_directive('rest_method', RestMethodDirective)
+    app.add_stylesheet('bootstrap.min.css')
+    app.add_stylesheet('api-site.css')
+    app.add_javascript('bootstrap.min.js')
+    app.add_javascript('api-site.js')
+    app.connect('doctree-read', resolve_rest_references)
+    return {'version': '0.1'}

api-ref/source/_static/api-site.css

@@ -0,0 +1,53 @@
+tt.literal {
+    padding: 2px 4px;
+    font-size: 90%;
+    color: #c7254e;
+    white-space: nowrap;
+    background-color: #f9f2f4;
+    border-radius: 4px;
+}
+
+pre {
+    display: block;
+    padding: 9.5px;
+    margin: 0 0 10px;
+    font-size: 13px;
+    line-height: 1.428571429;
+    color: #333;
+    word-break: break-all;
+    word-wrap: break-word;
+    background-color: #f5f5f5;
+    border: 1px solid #ccc;
+    border-radius: 4px;
+}
+
+tbody>tr:nth-child(odd)>td,
+tbody>tr:nth-child(odd)>th {
+    background-color: #f9f9f9;
+}
+
+table>thead>tr>th, table>tbody>tr>th, table>tfoot>tr>th, table>thead>tr>td, table>tbody>tr>td, table>tfoot>tr>td {
+    padding: 8px;
+    line-height: 1.428571429;
+    vertical-align: top;
+    border-top: 1px solid #ddd;
+}
+
+td>p {
+    margin: 0 0 0.5em;
+}
+
+div.document {
+    width: 80% !important;
+}
+
+@media (max-width: 1200px) {
+    div.document {
+        width: 960px !important;
+    }
+}
+
+.operation-grp {
+    padding-top: 0.5em;
+    padding-bottom: 1em;
+}

api-ref/source/_static/api-site.js

@@ -0,0 +1,95 @@
+(function() {
+
+  var pageCache;
+
+  $(document).ready(function() {
+    pageCache = $('.api-documentation').html();
+
+    // Show the proper JSON/XML example when toggled
+    $('.example-select').on('change', function(e) {
+      $(e.currentTarget).find(':selected').tab('show')
+    });
+
+    // Change the text on the expando buttons when appropriate
+    $('.api-detail')
+      .on('hide.bs.collapse', function(e) {
+        processButton(this, 'detail');
+      })
+      .on('show.bs.collapse', function(e) {
+        processButton(this, 'close');
+      });
+
+    // Wire up the search button
+    $('#search-btn').on('click', function(e) {
+      searchPage();
+    });
+
+    // Wire up the search box enter
+    $('#search-box').on('keydown', function(e) {
+      if (e.keyCode === 13) {
+        searchPage();
+        return false;
+      }
+    });
+  });
+
+  /**
+   * highlight terms based on the regex in the provided $element
+   */
+  function highlightTextNodes($element, regex) {
+    var markup = $element.html();
+
+    // Do regex replace
+    // Inject span with class of 'highlighted termX' for google style highlighting
+    $element.html(markup.replace(regex, '>$1<span class="highlight">$2</span>$3<'));
+  }
+
+  function searchPage() {
+    $(".api-documentation").html(pageCache);
+
+    //make sure that all div's are expanded/hidden accordingly
+    $('.api-detail.in').each(function (e) {
+      $(this).collapse('hide');
+    });
+
+    var startTime = new Date().getTime(),
+      searchTerm = $('#search-box').val();
+
+    // The regex is the secret, it prevents text within tag declarations to be affected
+    var regex = new RegExp(">([^<]*)?(" + searchTerm + ")([^>]*)?<", "ig");
+    highlightTextNodes($('.api-documentation'), regex);
+
+    // Once we've highlighted the node, lets expand any with a search match in them
+    $('.api-detail').each(function () {
+
+      var $elem = $(this);
+
+      if ($elem.html().indexOf('<span class="highlight">') !== -1) {
+        $elem.collapse('show');
+        processButton($elem, 'close');
+      }
+    });
+
+    // log the results
+    if (console.log) {
+      console.log("search completed in: " + ((new Date().getTime()) - startTime) + "ms");
+    }
+
+    $('.api-detail')
+      .on('hide.bs.collapse', function (e) {
+        processButton(this, 'detail');
+      })
+      .on('show.bs.collapse', function (e) {
+        processButton(this, 'close');
+      });
+  }
+
+  /**
+   * Helper function for setting the text, styles for expandos
+   */
+  function processButton(button, text) {
+    $('#' + $(button).attr('id') + '-btn').text(text)
+      .toggleClass('btn-info')
+      .toggleClass('btn-default');
+  }
+})();

api-ref/source/conf.py

@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# nova documentation build configuration file, created by
+# sphinx-quickstart on Sat May  1 15:17:47 2010.
+#
+# This file is execfile()d with the current directory set to
+# its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import os
+import subprocess
+import sys
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('../../'))
+sys.path.insert(0, os.path.abspath('../'))
+sys.path.insert(0, os.path.abspath('./'))
+
+# -- General configuration ----------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+
+extensions = [
+    'ext.rest_parameters',
+    'oslosphinx',
+]
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#
+# source_encoding = 'utf-8'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Compute API Reference'
+copyright = u'2010-present, OpenStack Foundation'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+from nova.version import version_info
+# The full version, including alpha/beta/rc tags.
+release = version_info.release_string()
+# The short X.Y version.
+version = version_info.version_string()
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+# today = ''
+# Else, today_fmt is used as the format for a strftime call.
+# today_fmt = '%B %d, %Y'
+
+# The reST default role (used for this markup: `text`) to use
+# for all documents.
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+add_module_names = False
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# -- Options for man page output ----------------------------------------------
+
+# Grouping the document tree for man pages.
+# List of tuples 'sourcefile', 'target', u'title', u'Authors name', 'manual'
+
+
+# -- Options for HTML output --------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  Major themes that come with
+# Sphinx are currently 'default' and 'sphinxdoc'.
+# html_theme_path = ["."]
+# html_theme = '_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+# html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+# html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+# html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+# html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+# html_last_updated_fmt = '%b %d, %Y'
+git_cmd = ["git", "log", "--pretty=format:'%ad, commit %h'", "--date=local",
+    "-n1"]
+html_last_updated_fmt = subprocess.Popen(
+    git_cmd, stdout=subprocess.PIPE).communicate()[0]
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+# html_use_modindex = True
+
+# If false, no index is generated.
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+# html_show_sourcelink = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+# html_use_opensearch = ''
+
+# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = ''
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'novadoc'
+
+
+# -- Options for LaTeX output -------------------------------------------------
+
+# The paper size ('letter' or 'a4').
+# latex_paper_size = 'letter'
+
+# The font size ('10pt', '11pt' or '12pt').
+# latex_font_size = '10pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass
+# [howto/manual]).
+latex_documents = [
+    ('index', 'Nova.tex', u'OpenStack Compute API Documentation',
+     u'OpenStack Foundation', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+# latex_use_parts = False
+
+# Additional stuff for the LaTeX preamble.
+# latex_preamble = ''
+
+# Documents to append as an appendix to all manuals.
+# latex_appendices = []
+
+# If false, no module index is generated.
+# latex_use_modindex = True