Add source code generation to conf.py

The sourcecode index was not generated previously for oslo-incubator.
This patch adds the code snippet, which generates autoindex for the
docstrings in each of the source files i nthe openstack folder. The
code snippet is based on the one that is used for Ceilometer.

The index.rst is modified to contain the correct autoindex reference.

The tox.ini was also updated with a 'docs' section.

Change-Id: I0afe2712246e4735e9ad97dd4bc2f98cd7d85613

doc/source/conf.py

@@ -1,9 +1,120 @@
 # -*- coding: utf-8 -*-
 
-import os
-import sys
+from __future__ import print_function
+
+import sys
+import os
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.abspath(os.path.join(BASE_DIR, "..", ".."))
+
+sys.path.insert(0, ROOT)
+sys.path.insert(0, BASE_DIR)
+
+# This is required for ReadTheDocs.org, but isn't a bad idea anyway.
+os.environ['DJANGO_SETTINGS_MODULE'] = 'openstack_dashboard.settings'
+
+
+def write_autodoc_index():
+
+    def find_autodoc_modules(module_name, sourcedir):
+        """Return a list of modules in the SOURCE directory."""
+        modlist = []
+        os.chdir(os.path.join(sourcedir, module_name))
+        print("SEARCHING %s" % sourcedir)
+        for root, dirs, files in os.walk("."):
+            for filename in files:
+                if filename.endswith(".py"):
+                    # remove the pieces of the root
+                    elements = root.split(os.path.sep)
+                    # replace the leading "." with the module name
+                    elements[0] = module_name
+                    # and get the base module name
+                    base, extension = os.path.splitext(filename)
+                    if not (base == "__init__"):
+                        elements.append(base)
+                    result = ".".join(elements)
+                    #print result
+                    modlist.append(result)
+        return modlist
+
+    RSTDIR = os.path.abspath(os.path.join(BASE_DIR, "sourcecode"))
+    SRCS = {'openstack': ROOT}
+
+    EXCLUDED_MODULES = ()
+    CURRENT_SOURCES = {}
+
+    if not(os.path.exists(RSTDIR)):
+        os.mkdir(RSTDIR)
+    CURRENT_SOURCES[RSTDIR] = ['autoindex.rst']
+
+    INDEXOUT = open(os.path.join(RSTDIR, "autoindex.rst"), "w")
+    INDEXOUT.write("=================\n")
+    INDEXOUT.write("Source Code Index\n")
+    INDEXOUT.write("=================\n")
+
+    for modulename, path in SRCS.items():
+        sys.stdout.write("Generating source documentation for %s\n" %
+                         modulename)
+        INDEXOUT.write("\n%s\n" % modulename.capitalize())
+        INDEXOUT.write("%s\n" % ("=" * len(modulename),))
+        INDEXOUT.write(".. toctree::\n")
+        INDEXOUT.write("   :maxdepth: 1\n")
+        INDEXOUT.write("\n")
+
+        MOD_DIR = os.path.join(RSTDIR, modulename)
+        CURRENT_SOURCES[MOD_DIR] = []
+        if not(os.path.exists(MOD_DIR)):
+            os.mkdir(MOD_DIR)
+        for module in find_autodoc_modules(modulename, path):
+            if any([module.startswith(exclude)
+                    for exclude
+                    in EXCLUDED_MODULES]):
+                print("Excluded module %s." % module)
+                continue
+            mod_path = os.path.join(path, *module.split("."))
+            generated_file = os.path.join(MOD_DIR, "%s.rst" % module)
+
+            INDEXOUT.write("   %s/%s\n" % (modulename, module))
+
+            # Find the __init__.py module if this is a directory
+            if os.path.isdir(mod_path):
+                source_file = ".".join((os.path.join(mod_path, "__init__"),
+                                        "py",))
+            else:
+                source_file = ".".join((os.path.join(mod_path), "py"))
+
+            CURRENT_SOURCES[MOD_DIR].append("%s.rst" % module)
+            # Only generate a new file if the source has changed or we don't
+            # have a doc file to begin with.
+            if not os.access(generated_file, os.F_OK) or \
+                    os.stat(generated_file).st_mtime < \
+                    os.stat(source_file).st_mtime:
+                print("Module %s updated, generating new documentation." \
+                      % module)
+                FILEOUT = open(generated_file, "w")
+                header = "The :mod:`%s` Module" % module
+                FILEOUT.write("%s\n" % ("=" * len(header),))
+                FILEOUT.write("%s\n" % header)
+                FILEOUT.write("%s\n" % ("=" * len(header),))
+                FILEOUT.write(".. automodule:: %s\n" % module)
+                FILEOUT.write("  :members:\n")
+                FILEOUT.write("  :undoc-members:\n")
+                FILEOUT.write("  :show-inheritance:\n")
+                FILEOUT.write("  :noindex:\n")
+                FILEOUT.close()
+
+    INDEXOUT.close()
+    # Delete auto-generated .rst files for sources which no longer exist
+    for directory, subdirs, files in list(os.walk(RSTDIR)):
+        for old_file in files:
+            if old_file not in CURRENT_SOURCES.get(directory, []):
+                print("Removing outdated file for %s" % old_file)
+                os.remove(os.path.join(directory, old_file))
+
+
+write_autodoc_index()
 
-sys.path.insert(0, os.path.abspath('../..'))
 # -- General configuration ----------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be

doc/source/index.rst

@@ -9,7 +9,7 @@ Contents
 .. toctree::
    :maxdepth: 1
 
-   api/autoindex
+   sourcecode/autoindex
 
 Indices and tables
 ==================

tox.ini

@@ -51,6 +51,9 @@ commands =
   # Use the lockutils wrapper to ensure that external locking works correctly
   python -m openstack.common.lockutils python setup.py test --coverage --testr-args='--concurrency=1 {posargs}'
 
+[testenv:docs]
+commands = python setup.py build_sphinx
+
 [testenv:venv]
 commands = {posargs}