Update V2 API documentation

This changeset adds a Sphinx extension for auto-generating
much of the documentation for a Pecan/WSME API from the
comments and docstrings in the source code. It also updates
the V2 API to include more documentation for API endpoints
and the data types used in the API, as well as sample data
for generating the JSON and XML examples in the output
documentation.

Change-Id: I1bde7805550aa86e9b64495b5c6034ec328479e5
Signed-off-by: Doug Hellmann <doug.hellmann@dreamhost.com>

ceilometer/api/controllers/v2.py

@@ -51,6 +51,11 @@ operation_kind = Enum(str, 'lt', 'le', 'eq', 'ne', 'ge', 'gt')
 
 
 class Query(Base):
+    """Query filter.
+    """
+
+    _op = None  # provide a default
+
     def get_op(self):
         return self._op or 'eq'
 
@@ -58,15 +63,27 @@ class Query(Base):
         self._op = value
 
     field = text
+    "The name of the field to test"
+
     #op = wsme.wsattr(operation_kind, default='eq')
     # this ^ doesn't seem to work.
     op = wsme.wsproperty(operation_kind, get_op, set_op)
+    "The comparison operator. Defaults to 'eq'."
+
     value = text
+    "The value to compare against the stored data"
 
     def __repr__(self):
         # for logging calls
         return '<Query %r %s %r>' % (self.field, self.op, self.value)
 
+    @classmethod
+    def sample(cls):
+        return cls(field='resource_id',
+                   op='eq',
+                   value='bd9431c1-8d69-4ad3-803a-8d4a6b89fd36',
+                   )
+
 
 def _query_to_kwargs(query, db_func):
     # TODO(dhellmann): This function needs tests of its own.
@@ -184,17 +201,44 @@ def _flatten_metadata(metadata):
 
 
 class Sample(Base):
+    """A single measurement for a given meter and resource.
+    """
+
     source = text
+    "An identity source ID"
+
     counter_name = text
+    "The name of the meter"
+    # FIXME(dhellmann): Make this meter_name?
+
     counter_type = text
+    "The type of the meter (see :ref:`measurements`)"
+    # FIXME(dhellmann): Make this meter_type?
+
     counter_unit = text
+    "The unit of measure for the value in counter_volume"
+    # FIXME(dhellmann): Make this meter_unit?
+
     counter_volume = float
+    "The actual measured value"
+
     user_id = text
+    "The ID of the user who last triggered an update to the resource"
+
     project_id = text
+    "The ID of the project or tenant that owns the resource"
+
     resource_id = text
+    "The ID of the :class:`Resource` for which the measurements are taken"
+
     timestamp = datetime.datetime
+    "UTC date and time when the measurement was made"
+
     resource_metadata = {text: text}
+    "Arbitrary metadata associated with the resource"
+
     message_id = text
+    "A unique identifier for the sample"
 
     def __init__(self, counter_volume=None, resource_metadata={}, **kwds):
         if counter_volume is not None:
@@ -204,16 +248,50 @@ class Sample(Base):
                                      resource_metadata=resource_metadata,
                                      **kwds)
 
+    @classmethod
+    def sample(cls):
+        return cls(source='openstack',
+                   counter_name='instance',
+                   counter_type='gauge',
+                   counter_unit='instance',
+                   counter_volume=1,
+                   resource_id='bd9431c1-8d69-4ad3-803a-8d4a6b89fd36',
+                   project_id='35b17138-b364-4e6a-a131-8f3099c5be68',
+                   user_id='efd87807-12d2-4b38-9c70-5f5c2ac427ff',
+                   timestamp=datetime.datetime.utcnow(),
+                   metadata={'name1': 'value1',
+                             'name2': 'value2'},
+                   message_id='5460acce-4fd6-480d-ab18-9735ec7b1996',
+                   )
+
 
 class Statistics(Base):
+    """Computed statistics for a query.
+    """
+
     min = float
+    "The minimum volume seen in the data"
+
     max = float
+    "The maximum volume seen in the data"
+
     avg = float
+    "The average of all of the volume values seen in the data"
+
     sum = float
+    "The total of all of the volume values seen in the data"
+
     count = int
+    "The number of samples seen"
+
     duration = float
+    "The difference, in minutes, between the oldest and newest timestamp"
+
     duration_start = datetime.datetime
+    "UTC date and time of the earliest timestamp, or the query start time"
+
     duration_end = datetime.datetime
+    "UTC date and time of the oldest timestamp, or the query end time"
 
     def __init__(self, start_timestamp=None, end_timestamp=None, **kwds):
         super(Statistics, self).__init__(**kwds)
@@ -250,9 +328,22 @@ class Statistics(Base):
             # it is not available in Python 2.6.
             diff = self.duration_end - self.duration_start
             self.duration = (diff.seconds + (diff.days * 24 * 60 ** 2)) / 60
+            # FIXME(dhellmann): Shouldn't this value be returned in
+            # seconds, or something even smaller?
         else:
             self.duration_start = self.duration_end = self.duration = None
 
+    @classmethod
+    def sample(cls):
+        return cls(min=1,
+                   max=9,
+                   avg=4.5,
+                   sum=45,
+                   count=10,
+                   duration_start=datetime.datetime(2013, 1, 4, 16, 42),
+                   duration_end=datetime.datetime(2013, 1, 4, 16, 47),
+                   )
+
 
 class MeterController(RestController):
     """Manages operations on a single meter.
@@ -267,7 +358,9 @@ class MeterController(RestController):
 
     @wsme_pecan.wsexpose([Sample], [Query])
     def get_all(self, q=[]):
-        """Return all events for the meter.
+        """Return sample data for the meter.
+
+        :param q: Filter rules for the data to be returned.
         """
         kwargs = _query_to_kwargs(q, storage.EventFilter.__init__)
         kwargs['meter'] = self._id
@@ -299,12 +392,37 @@ class MeterController(RestController):
 
 
 class Meter(Base):
+    """One category of measurements.
+    """
+
     name = text
+    "The unique name for the meter"
+
+    # FIXME(dhellmann): Make this an enum?
     type = text
+    "The meter type (see :ref:`measurements`)"
+
     unit = text
+    "The unit of measure"
+
     resource_id = text
+    "The ID of the :class:`Resource` for which the measurements are taken"
+
     project_id = text
+    "The ID of the project or tenant that owns the resource"
+
     user_id = text
+    "The ID of the user who last triggered an update to the resource"
+
+    @classmethod
+    def sample(cls):
+        return cls(name='instance',
+                   type='gauge',
+                   unit='instance',
+                   resource_id='bd9431c1-8d69-4ad3-803a-8d4a6b89fd36',
+                   project_id='35b17138-b364-4e6a-a131-8f3099c5be68',
+                   user_id='efd87807-12d2-4b38-9c70-5f5c2ac427ff',
+                   )
 
 
 class MetersController(RestController):
@@ -316,46 +434,67 @@ class MetersController(RestController):
 
     @wsme_pecan.wsexpose([Meter], [Query])
     def get_all(self, q=[]):
+        """Return all known meters, based on the data recorded so far.
+
+        :param q: Filter rules for the meters to be returned.
+        """
         kwargs = _query_to_kwargs(q, request.storage_conn.get_meters)
         return [Meter(**m)
                 for m in request.storage_conn.get_meters(**kwargs)]
 
 
 class Resource(Base):
+    """An externally defined object for which samples have been received.
+    """
+
     resource_id = text
+    "The unique identifier for the resource"
+
     project_id = text
+    "The ID of the owning project or tenant"
+
     user_id = text
+    "The ID of the user who created the resource or updated it last"
+
     timestamp = datetime.datetime
+    "UTC date and time of the last update to any meter for the resource"
+
     metadata = {text: text}
+    "Arbitrary metadata associated with the resource"
 
     def __init__(self, metadata={}, **kwds):
         metadata = _flatten_metadata(metadata)
         super(Resource, self).__init__(metadata=metadata, **kwds)
 
-
+    @classmethod
-class ResourceController(RestController):
+    def sample(cls):
-    """Manages operations on a single resource.
+        return cls(resource_id='bd9431c1-8d69-4ad3-803a-8d4a6b89fd36',
-    """
+                   project_id='35b17138-b364-4e6a-a131-8f3099c5be68',
-
+                   user_id='efd87807-12d2-4b38-9c70-5f5c2ac427ff',
-    def __init__(self, resource_id):
+                   timestamp=datetime.datetime.utcnow(),
-        request.context['resource_id'] = resource_id
+                   metadata={'name1': 'value1',
-
+                             'name2': 'value2'},
-    @wsme_pecan.wsexpose([Resource])
+                   )
-    def get_all(self):
-            r = request.storage_conn.get_resources(
-                resource=request.context.get('resource_id'))[0]
-            return Resource(**r)
 
 
 class ResourcesController(RestController):
     """Works on resources."""
 
-    @pecan.expose()
+    @wsme_pecan.wsexpose(Resource, unicode)
-    def _lookup(self, resource_id, *remainder):
+    def get_one(self, resource_id):
-        return ResourceController(resource_id), remainder
+        """Retrieve details about one resource.
+
+        :param resource_id: The UUID of the resource.
+        """
+        r = request.storage_conn.get_resources(resource=resource_id)[0]
+        return Resource(**r)
 
     @wsme_pecan.wsexpose([Resource], [Query])
     def get_all(self, q=[]):
+        """Retrieve definitions of all of the resources.
+
+        :param q: Filter rules for the resources to be returned.
+        """
         kwargs = _query_to_kwargs(q, request.storage_conn.get_resources)
         resources = [
             Resource(**r)

doc/source/ceilext/api.py

@@ -0,0 +1,189 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2013 New Dream Network, LLC (DreamHost)
+#
+# Author: Doug Hellmann <doug.hellmann@dreamhost.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Sphinx extension for automatically generating API documentation
+from Pecan controllers exposed through WSME.
+
+"""
+import inspect
+
+from docutils import nodes
+from docutils.parsers import rst
+from docutils.statemachine import ViewList
+
+from sphinx.util.nodes import nested_parse_with_titles
+from sphinx.util.docstrings import prepare_docstring
+
+import wsme.types
+
+
+def import_object(import_name):
+    """Import the named object and return it.
+
+    The name should be formatted as package.module:obj.
+    """
+    module_name, expr = import_name.split(':', 1)
+    mod = __import__(module_name)
+    mod = reduce(getattr, module_name.split('.')[1:], mod)
+    globals = __builtins__
+    if not isinstance(globals, dict):
+        globals = globals.__dict__
+    return eval(expr, globals, mod.__dict__)
+
+
+def http_directive(method, path, content):
+    """Build an HTTP directive for documenting a single URL.
+
+    :param method: HTTP method ('get', 'post', etc.)
+    :param path: URL
+    :param content: Text describing the endpoint.
+    """
+    method = method.lower().strip()
+    if isinstance(content, basestring):
+        content = content.splitlines()
+    yield ''
+    yield '.. http:{method}:: {path}'.format(**locals())
+    yield ''
+    for line in content:
+        yield '   ' + line
+    yield ''
+
+
+def datatypename(datatype):
+    """Return the formatted name of the data type.
+
+    Derived from wsmeext.sphinxext.datatypename.
+    """
+    if isinstance(datatype, wsme.types.DictType):
+        return 'dict(%s: %s)' % (datatypename(datatype.key_type),
+                                 datatypename(datatype.value_type))
+    if isinstance(datatype, wsme.types.ArrayType):
+        return 'list(%s)' % datatypename(datatype.item_type)
+    if isinstance(datatype, wsme.types.UserType):
+        return ':class:`%s`' % datatype.name
+    if isinstance(datatype, wsme.types.Base) or hasattr(datatype, '__name__'):
+        return ':class:`%s`' % datatype.__name__
+    return datatype.__name__
+
+
+class RESTControllerDirective(rst.Directive):
+
+    required_arguments = 1
+    option_spec = {
+        'webprefix': rst.directives.unchanged,
+    }
+    has_content = True
+
+    def make_rst_for_method(self, path, method):
+        docstring = prepare_docstring((method.__doc__ or '').rstrip('\n'))
+        blank_line = docstring[-1]
+        docstring = docstring[:-1]  # remove blank line appended automatically
+
+        funcdef = method._wsme_definition
+
+        # Add the parameter type information. Assumes that the
+        # developer has provided descriptions of the parameters.
+        for arg in funcdef.arguments:
+            docstring.append(':type %s: %s' %
+                             (arg.name, datatypename(arg.datatype)))
+
+        # Add the return type
+        if funcdef.return_type:
+            return_type = datatypename(funcdef.return_type)
+            docstring.append(':return type: %s' % return_type)
+
+        # restore the blank line added as a spacer
+        docstring.append(blank_line)
+
+        directive = http_directive('get', path, docstring)
+        for line in directive:
+            yield line
+
+    def make_rst_for_controller(self, path_prefix, controller):
+        env = self.state.document.settings.env
+        app = env.app
+
+        controller_path = path_prefix.rstrip('/') + '/'
+
+        # Some of the controllers are instantiated dynamically, so
+        # we need to look at their constructor arguments to see
+        # what parameters are needed and include them in the
+        # URL. For now, we only ever want one at a time.
+        try:
+            argspec = inspect.getargspec(controller.__init__)
+        except TypeError:
+            # The default __init__ for object is a "slot wrapper" not
+            # a method, so we can't inspect it. It doesn't take any
+            # arguments, though, so just knowing that we didn't
+            # override __init__ helps us build the controller path
+            # correctly.
+            pass
+        else:
+            if len(argspec[0]) > 1:
+                first_arg_name = argspec[0][1]
+                controller_path += '(' + first_arg_name + ')/'
+
+        if hasattr(controller, 'get_all') and controller.get_all.exposed:
+            app.info('  Method: get_all')
+            for line in self.make_rst_for_method(controller_path,
+                                                 controller.get_all):
+                yield line
+
+        if hasattr(controller, 'get_one') and controller.get_one.exposed:
+            app.info('  Method: %s' % controller.get_one)
+            funcdef = controller.get_one._wsme_definition
+            first_arg_name = funcdef.arguments[0].name
+            path = controller_path + '(' + first_arg_name + ')/'
+            for line in self.make_rst_for_method(
+                    path,
+                    controller.get_one):
+                yield line
+
+        # Look for exposed custom methods
+        for name in sorted(controller._custom_actions.keys()):
+            app.info('  Method: %s' % name)
+            method = getattr(controller, name)
+            path = controller_path + name + '/'
+            for line in self.make_rst_for_method(path, method):
+                yield line
+
+    def run(self):
+        env = self.state.document.settings.env
+        app = env.app
+        controller_id = self.arguments[0]
+        app.info('found root-controller %s' % controller_id)
+
+        result = ViewList()
+        controller = import_object(self.arguments[0])
+
+        for line in self.make_rst_for_controller(
+                self.options.get('webprefix', '/'),
+                controller):
+            app.info('ADDING: %r' % line)
+            result.append(line, '<' + __name__ + '>')
+
+        node = nodes.section()
+        # necessary so that the child nodes get the right source/line set
+        node.document = self.state.document
+        nested_parse_with_titles(self.state, result, node)
+
+        return node.children
+
+
+def setup(app):
+    app.info('Initializing %s' % __name__)
+    app.add_directive('rest-controller', RESTControllerDirective)

doc/source/conf.py

@@ -19,6 +19,7 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 ROOT = os.path.abspath(os.path.join(BASE_DIR, "..", ".."))
 
 sys.path.insert(0, ROOT)
+sys.path.insert(0, BASE_DIR)
 
 # This is required for ReadTheDocs.org, but isn't a bad idea anyway.
 os.environ['DJANGO_SETTINGS_MODULE'] = 'openstack_dashboard.settings'
@@ -146,7 +147,8 @@ extensions = ['sphinx.ext.autodoc',
               'wsmeext.sphinxext',
               'sphinx.ext.coverage',
               'sphinx.ext.pngmath',
-              'sphinx.ext.viewcode']
+              'sphinx.ext.viewcode',
+              'ceilext.api']
 
 wsme_protocols = ['restjson', 'restxml']
 

doc/source/webapi/v2.rst

@@ -2,11 +2,41 @@
  V2 Web API
 ============
 
-.. default-domain:: wsme
+Resources
+=========
 
-.. root:: ceilometer.api.controllers.root.RootController
+.. rest-controller:: ceilometer.api.controllers.v2:ResourcesController
-   :webpath:
+   :webprefix: /v2/resources
 
-.. autotype:: ceilometer.api.controllers.v2.Source
+.. autotype:: ceilometer.api.controllers.v2.Resource
+   :members:
 
-.. service:: /v2/sources
+Meters
+======
+
+.. rest-controller:: ceilometer.api.controllers.v2:MetersController
+   :webprefix: /v2/meters
+
+.. rest-controller:: ceilometer.api.controllers.v2:MeterController
+   :webprefix: /v2/meters
+
+Samples and Statistics
+======================
+
+.. autotype:: ceilometer.api.controllers.v2.Meter
+   :members:
+
+.. autotype:: ceilometer.api.controllers.v2.Sample
+   :members:
+
+.. autotype:: ceilometer.api.controllers.v2.Statistics
+   :members:
+
+Filtering Queries
+=================
+
+Many of the endpoints above accecpt a query filter argument, which
+should be a list of Query data structures:
+
+.. autotype:: ceilometer.api.controllers.v2.Query
+   :members: