Build cleaner and easier to read docs for dynamic surfaces.

Reviewed in http://codereview.appspot.com/6376043/.
2012-07-09 16:46:02 -04:00
parent bf14cefd46
commit 81d92cc73f
6 changed files with 366 additions and 67 deletions
--- a/apiclient/discovery.py
+++ b/apiclient/discovery.py
@@ -74,7 +74,7 @@ STACK_QUERY_PARAMETERS = ['trace', 'pp', 'userip', 'strict']
 RESERVED_WORDS = ['and', 'assert', 'break', 'class', 'continue', 'def', 'del',
                  'elif', 'else', 'except', 'exec', 'finally', 'for', 'from',
                  'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or',
-                  'pass', 'print', 'raise', 'return', 'try', 'while' ]
+                  'pass', 'print', 'raise', 'return', 'try', 'while', 'body']


 def fix_method_name(name):
@@ -396,7 +396,9 @@ def _createResource(http, baseUrl, model, requestBuilder,
        methodDesc['parameters']['body']['type'] = 'object'
    if 'mediaUpload' in methodDesc:
      methodDesc['parameters']['media_body'] = {
-          'description': 'The filename of the media request body.',
+          'description':
+            'The filename of the media request body, or an instance of a '
+            'MediaUpload object.',
          'type': 'string',
          'required': False,
          }
@@ -596,9 +598,20 @@ def _createResource(http, baseUrl, model, requestBuilder,

    # Skip undocumented params and params common to all methods.
    skip_parameters = rootDesc.get('parameters', {}).keys()
-    skip_parameters.append(STACK_QUERY_PARAMETERS)
+    skip_parameters.extend(STACK_QUERY_PARAMETERS)

-    for arg in argmap.iterkeys():
+    all_args = argmap.keys()
+    args_ordered = [key2param(s) for s in methodDesc.get('parameterOrder', [])]
+
+    # Move body to the front of the line.
+    if 'body' in all_args:
+      args_ordered.append('body')
+
+    for name in all_args:
+      if name not in args_ordered:
+        args_ordered.append(name)
+
+    for arg in args_ordered:
      if arg in skip_parameters:
        continue

@@ -653,13 +666,13 @@ def _createResource(http, baseUrl, model, requestBuilder,
    def methodNext(self, previous_request, previous_response):
      """Retrieves the next page of results.

-      Args:
-        previous_request: The request for the previous page.
-        previous_response: The response from the request for the previous page.
+Args:
+  previous_request: The request for the previous page. (required)
+  previous_response: The response from the request for the previous page. (required)

-      Returns:
-        A request object that you can call 'execute()' on to request the next
-        page. Returns None if there are no more items in the collection.
+Returns:
+  A request object that you can call 'execute()' on to request the next
+  page. Returns None if there are no more items in the collection.
      """
      # Retrieve nextPageToken from previous_response
      # Use as pageToken in previous_request to create new request.
--- a/describe.py
+++ b/describe.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright 2007 Google Inc.
+# Copyright 2012 Google Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,47 +14,332 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+"""Create documentation for generate API surfaces.
+
+Command-line tool that creates documentation for all APIs listed in discovery.
+The documentation is generated from a combination of the discovery document and
+the generated API surface itself.
+"""
+
 __author__ = 'jcgregorio@google.com (Joe Gregorio)'

+import json
 import os
-import pydoc
 import re
 import sys
 import httplib2

-from oauth2client.anyjson import simplejson
+from string import Template
+
 from apiclient.discovery import build
+from oauth2client.anyjson import simplejson
+import uritemplate
+

 BASE = 'docs/dyn'

-def document(resource, path):
-  print path
+CSS = """<style>
+
+body, h1, h2, h3, div, span, p, pre, a {
+  margin: 0;
+  padding: 0;
+  border: 0;
+  font-weight: inherit;
+  font-style: inherit;
+  font-size: 100%;
+  font-family: inherit;
+  vertical-align: baseline;
+}
+
+body {
+  font-size: 13px;
+  padding: 1em;
+}
+
+h1 {
+  font-size: 26px;
+  margin-bottom: 1em;
+}
+
+h2 {
+  font-size: 24px;
+  margin-bottom: 1em;
+}
+
+h3 {
+  font-size: 20px;
+  margin-bottom: 1em;
+  margin-top: 1em;
+}
+
+pre, code {
+  line-height: 1.5;
+  font-family: Monaco, 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', 'Lucida Console', monospace;
+}
+
+pre {
+  margin-top: 0.5em;
+}
+
+h1, h2, h3, p {
+  font-family: Arial, sans serif;
+}
+
+h1, h2, h3 {
+  border-bottom: solid #CCC 1px;
+}
+
+.toc_element {
+  margin-top: 0.5em;
+}
+
+.firstline {
+  margin-left: 2 em;
+}
+
+.method  {
+  margin-top: 1em;
+  border: solid 1px #CCC;
+  padding: 1em;
+  background: #EEE;
+}
+
+.details {
+  font-weight: bold;
+  font-size: 14px;
+}
+
+</style>
+"""
+
+DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
+                 '{api}/{apiVersion}/rest')
+
+METHOD_TEMPLATE = """<div class="method">
+    <code class="details" id="$name">$name($params)</code>
+  <pre>$doc</pre>
+</div>
+"""
+
+COLLECTION_LINK = """<p class="toc_element">
+  <code><a href="$href">$name()</a></code>
+</p>
+<p class="firstline">Returns the $name Resource.</p>
+"""
+
+METHOD_LINK = """<p class="toc_element">
+  <code><a href="#$name">$name($params)</a></code></p>
+<p class="firstline">$firstline</p>"""
+
+
+def safe_version(version):
+  """Create a safe version of the verion string.
+
+  Needed so that we can distinguish between versions
+  and sub-collections in URIs. I.e. we don't want
+  adsense_v1.1 to refer to the '1' collection in the v1
+  version of the adsense api.
+
+  Args:
+    version: string, The version string.
+  Returns:
+    The string with '.' replaced with '_'.
+  """
+
+  return version.replace('.', '_')
+
+
+def unsafe_version(version):
+  """Undoes what safe_version() does.
+
+  See safe_version() for the details.
+
+
+  Args:
+    version: string, The safe version string.
+  Returns:
+    The string with '_' replaced with '.'.
+  """
+
+  return version.replace('_', '.')
+
+
+def method_params(doc):
+  """Document the parameters of a method.
+
+  Args:
+    doc: string, The method's docstring.
+
+  Returns:
+    The method signature as a string.
+  """
+  doclines = doc.splitlines()
+  if 'Args:' in doclines:
+    begin = doclines.index('Args:')
+    if 'Returns:' in doclines[begin+1:]:
+      end = doclines.index('Returns:', begin)
+      args = doclines[begin+1: end]
+    else:
+      args = doclines[begin+1:]
+
+    parameters = []
+    for line in args:
+      m = re.search('^\s+([a-zA-Z0-9_]+): (.*)', line)
+      if m is None:
+        continue
+      pname = m.group(1)
+      desc = m.group(2)
+      if '(required)' not in desc:
+        pname = pname + '=None'
+      parameters.append(pname)
+    parameters = ', '.join(parameters)
+  else:
+    parameters = ''
+  return parameters
+
+
+def method(name, doc):
+  """Documents an individual method.
+
+  Args:
+    name: string, Name of the method.
+    doc: string, The methods docstring.
+  """
+
+  params = method_params(doc)
+  return Template(METHOD_TEMPLATE).substitute(name=name, params=params, doc=doc)
+
+
+def breadcrumbs(path, root_discovery):
+  """Create the breadcrumb trail to this page of documentation.
+
+  Args:
+    path: string, Dot separated name of the resource.
+    root_discovery: Deserialized discovery document.
+
+  Returns:
+    HTML with links to each of the parent resources of this resource.
+  """
+  parts = path.split('.')
+
+  crumbs = []
+  accumulated = []
+
+  for i, p in enumerate(parts):
+    prefix = '.'.join(accumulated)
+    # The first time through prefix will be [], so we avoid adding in a
+    # superfluous '.' to prefix.
+    if prefix:
+      prefix += '.'
+    display = p
+    if i == 0:
+      display = root_discovery.get('title', display)
+    crumbs.append('<a href="%s.html">%s</a>' % (prefix + p, display))
+    accumulated.append(p)
+
+  return ' . '.join(crumbs)
+
+
+def document_collection(resource, path, root_discovery, discovery, css=CSS):
+  """Document a single collection in an API.
+
+  Args:
+    resource: Collection or service being documented.
+    path: string, Dot separated name of the resource.
+    root_discovery: Deserialized discovery document.
+    discovery: Deserialized discovery document, but just the portion that
+      describes the resource.
+    css: string, The CSS to include in the generated file.
+  """
  collections = []
+  methods = []
+  resource_name = path.split('.')[-2]
+  html = [
+      '<html><body>',
+      css,
+      '<h1>%s</h1>' % breadcrumbs(path[:-1], root_discovery),
+      '<h2>Instance Methods</h2>'
+      ]
+
+  # Which methods are for collections.
  for name in dir(resource):
-    if not "_" in name and callable(getattr(resource, name)) and hasattr(
-          getattr(resource, name), '__is_resource__'):
-      collections.append(name)
+    if not name.startswith('_') and callable(getattr(resource, name)):
+      if hasattr(getattr(resource, name), '__is_resource__'):
+        collections.append(name)
+      else:
+        methods.append(name)

-  obj, name = pydoc.resolve(type(resource))
-  page = pydoc.html.page(
-      pydoc.describe(obj), pydoc.html.document(obj, name))

-  for name in collections:
-    page = re.sub('strong>(%s)<' % name, r'strong><a href="%s">\1</a><' % (path + name + ".html"), page)
-  for name in collections:
-    document(getattr(resource, name)(), path + name + ".")
+  # TOC
+  if collections:
+    for name in collections:
+      if not name.startswith('_') and callable(getattr(resource, name)):
+        href = path + name + '.html'
+        html.append(Template(COLLECTION_LINK).substitute(href=href, name=name))
+
+  if methods:
+    for name in methods:
+      if not name.startswith('_') and callable(getattr(resource, name)):
+        doc = getattr(resource, name).__doc__
+        params = method_params(doc)
+        firstline = doc.splitlines()[0]
+        html.append(Template(METHOD_LINK).substitute(
+            name=name, params=params, firstline=firstline))
+
+  if methods:
+    html.append('<h3>Method Details</h3>')
+    for name in methods:
+      dname = name.rsplit('_')[0]
+      html.append(method(name, getattr(resource, name).__doc__))
+
+  html.append('</body></html>')
+
+  return '\n'.join(html)
+
+
+def document_collection_recursive(resource, path, root_discovery, discovery):
+
+  html = document_collection(resource, path, root_discovery, discovery)

  f = open(os.path.join(BASE, path + 'html'), 'w')
-  f.write(page)
+  f.write(html)
  f.close()

+  for name in dir(resource):
+    if (not name.startswith('_')
+        and callable(getattr(resource, name))
+        and hasattr(getattr(resource, name), '__is_resource__')):
+      dname = name.rsplit('_')[0]
+      collection = getattr(resource, name)()
+      document_collection_recursive(collection, path + name + '.', root_discovery,
+               discovery['resources'].get(dname, {}))
+
 def document_api(name, version):
+  """Document the given API.
+
+  Args:
+    name: string, Name of the API.
+    version: string, Version of the API.
+  """
  service = build(name, version)
-  document(service, '%s.%s.' % (name, version))
+  response, content = http.request(
+      uritemplate.expand(
+          DISCOVERY_URI, {
+              'api': name,
+              'apiVersion': version})
+          )
+  discovery = json.loads(content)
+
+  version = safe_version(version)
+
+  document_collection_recursive(
+      service, '%s_%s.' % (name, version), discovery, discovery)
+

 if __name__ == '__main__':
  http = httplib2.Http()
-  resp, content = http.request('https://www.googleapis.com/discovery/v0.3/directory?preferred=true')
+  resp, content = http.request(
+      'https://www.googleapis.com/discovery/v1/apis?preferred=true')
  if resp.status == 200:
    directory = simplejson.loads(content)['items']
    for api in directory:
--- a/samples/api-python-client-doc/embed.html
+++ b/samples/api-python-client-doc/embed.html
@@ -68,7 +68,7 @@
      <tr>
        <td><img class=icon src="{{ item.icons.x16 }}"/> {{ item.title }}</td>
        <td><a target=_top href="{{ item.documentationLink }}">Documentation</a></td>
-        <td><a target=_top href="/{{ item.name }}/{{ item.version }}">PyDoc</a></td>
+        <td><a target=_top href="/{{ item.name }}_{{ item.safe_version }}.html">PyDoc</a></td>
        <td>{{ item.name }}</td>
        <td>{{ item.version}}</td>
      </tr>
--- a/samples/api-python-client-doc/gadget.html
+++ b/samples/api-python-client-doc/gadget.html
@@ -8,7 +8,7 @@
      <tr> 
        <td><img style="width: 16px; height: 16px" src="{{ item.icons.x16 }}"/> {{ item.name }} </td>
        <td><a href="{{ item.documentationLink }}">Documentation</a></td>
-        <td><a href="/{{ item.name }}/{{ item.version }}">PyDoc</a></td>
+        <td><a href="/{{ item.name }}_{{ item.safe_version }}.html">PyDoc</a></td>
      </tr> 
      {% endfor %}
    </table>
--- a/samples/api-python-client-doc/index.html
+++ b/samples/api-python-client-doc/index.html
@@ -69,7 +69,7 @@
      <tr>
        <td><img class=icon src="{{ item.icons.x16 }}"/> {{ item.title }}</td>
        <td><a href="{{ item.documentationLink }}">Documentation</a></td>
-        <td><a href="/{{ item.name }}/{{ item.version }}">PyDoc</a></td>
+        <td><a href="/{{ item.name }}_{{ item.safe_version }}.html">PyDoc</a></td>
        <td>{{ item.name }}</td>
        <td>{{ item.version}}</td>
      </tr>
--- a/samples/api-python-client-doc/main.py
+++ b/samples/api-python-client-doc/main.py
@@ -26,10 +26,14 @@ __author__ = 'jcgregorio@google.com (Joe Gregorio)'

 import httplib2
 import inspect
+import logging
 import os
 import pydoc
 import re

+import describe
+import uritemplate
+
 from apiclient import discovery
 from apiclient.errors import HttpError
 from google.appengine.api import memcache
@@ -50,6 +54,9 @@ def get_directory_doc():
    uri += ('&userIp=' + ip)
  resp, content = http.request(uri)
  directory = simplejson.loads(content)['items']
+  for item in directory:
+    item['title'] = item.get('title', item.get('description', ''))
+    item['safe_version'] = describe.safe_version(item['version'])
  return directory


@@ -59,8 +66,6 @@ class MainHandler(webapp.RequestHandler):

  def get(self):
    directory = get_directory_doc()
-    for item in directory:
-      item['title'] = item.get('title', item.get('description', ''))
    path = os.path.join(os.path.dirname(__file__), 'index.html')
    self.response.out.write(
        template.render(
@@ -73,8 +78,6 @@ class GadgetHandler(webapp.RequestHandler):

  def get(self):
    directory = get_directory_doc()
-    for item in directory:
-      item['title'] = item.get('title', item.get('description', ''))
    path = os.path.join(os.path.dirname(__file__), 'gadget.html')
    self.response.out.write(
        template.render(
@@ -88,8 +91,6 @@ class EmbedHandler(webapp.RequestHandler):

  def get(self):
    directory = get_directory_doc()
-    for item in directory:
-      item['title'] = item.get('title', item.get('description', ''))
    path = os.path.join(os.path.dirname(__file__), 'embed.html')
    self.response.out.write(
        template.render(
@@ -97,54 +98,54 @@ class EmbedHandler(webapp.RequestHandler):
                   }))


-def _render(resource):
-  """Use pydoc helpers on an instance to generate the help documentation.
-  """
-  obj, name = pydoc.resolve(type(resource))
-  return pydoc.html.page(
-      pydoc.describe(obj), pydoc.html.document(obj, name))
-
-
 class ResourceHandler(webapp.RequestHandler):
  """Handles serving the PyDoc for a given collection.
  """

  def get(self, service_name, version, collection):
+
+    real_version = describe.unsafe_version(version)
+
+    logging.info('%s %s %s', service_name, version, collection)
    http = httplib2.Http(memcache)
    try:
-      resource = discovery.build(service_name, version, http=http)
+      resource = discovery.build(service_name, real_version, http=http)
    except:
+      logging.error('Failed to build service.')
      return self.error(404)
+
+    DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
+      '{api}/{apiVersion}/rest')
+    response, content = http.request(
+        uritemplate.expand(
+            DISCOVERY_URI, {
+                'api': service_name,
+                'apiVersion': real_version})
+            )
+    root_discovery = simplejson.loads(content)
+    collection_discovery = root_discovery
+
    # descend the object path
    if collection:
      try:
-        path = collection.split('/')
+        path = collection.split('.')
        if path:
          for method in path:
            resource = getattr(resource, method)()
+            collection_discovery = collection_discovery['resources'][method]
      except:
+        logging.error('Failed to parse the collections.')
        return self.error(404)
+    logging.info('Built everything successfully so far.')

-    page = _render(resource)
+    path = '%s_%s.' % (service_name, version)
+    if collection:
+      path += '.'.join(collection.split('/'))
+      path += '.'

-    collections = []
-    for name in dir(resource):
-      if not "_" in name and callable(getattr(resource, name)) and hasattr(
-          getattr(resource, name), '__is_resource__'):
-        collections.append(name)
+    page = describe.document_collection(
+        resource, path, root_discovery, collection_discovery)

-    if collection is None:
-      collection_path = ''
-    else:
-      collection_path = collection + '/'
-    for name in collections:
-      page = re.sub('strong>(%s)<' % name,
-          r'strong><a href="/%s/%s/%s">\1</a><' % (
-          service_name, version, collection_path + name), page)
-
-    # TODO(jcgregorio) breadcrumbs
-    # TODO(jcgregorio) sample code?
-    page = re.sub('<p>', r'<a href="/">Home</a><p>', page, 1)
    self.response.out.write(page)


@@ -154,9 +155,9 @@ def main():
      (r'/', MainHandler),
      (r'/_gadget/', GadgetHandler),
      (r'/_embed/', EmbedHandler),
-      (r'/([^\/]*)/([^\/]*)(?:/(.*))?', ResourceHandler),
+      (r'/([^_]+)_([^\.]+)(?:\.(.*))?\.html$', ResourceHandler),
      ],
-      debug=False)
+      debug=True)
  util.run_wsgi_app(application)