Automatically generate Samples wiki page from README files.

Reviewed in http://codereview.appspot.com/5900069/.

Index: samples-index.py
===================================================================
new file mode 100644

Makefile

@@ -19,6 +19,11 @@ coverage:
 docs:
 	cd docs; ./build.sh
 	python describe.py
+	python samples-index.py ../google-api-python-client.wiki/SampleApps.wiki
+
+.PHONY: wiki
+wiki:
+	python samples-index.py > ../google-api-python-client.wiki/SampleApps.wiki
 
 .PHONY: prerelease
 prerelease:

samples-index.py

@@ -0,0 +1,229 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2012 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Build wiki page with a list of all samples.
+
+The information for the wiki page is built from data found in all the README
+files in the samples. The format of the README file is:
+
+
+   Description is everything up to the first blank line.
+
+   api: plus  (Used to look up the long name in discovery).
+   keywords: appengine (such as appengine, oauth2, cmdline)
+
+   The rest of the file is ignored when it comes to building the index.
+"""
+
+import httplib2
+import itertools
+import json
+import os
+import re
+
+http = httplib2.Http('.cache')
+r, c =  http.request('https://www.googleapis.com/discovery/v1/apis')
+if r.status != 200:
+  raise ValueError('Received non-200 response when retrieving Discovery document.')
+
+# Dictionary mapping api names to their discovery description.
+DIRECTORY = {}
+for item in json.loads(c)['items']:
+  if item['preferred']:
+    DIRECTORY[item['name']] = item
+
+# A list of valid keywords. Should not be taken as complete, add to
+# this list as needed.
+KEYWORDS = {
+    'appengine': 'Google App Engine',
+    'oauth2': 'OAuth 2.0',
+    'cmdline': 'Command-line',
+    'django': 'Django',
+    'threading': 'Threading',
+    'pagination': 'Pagination'
+    }
+
+
+def get_lines(name, lines):
+  """Return lines that begin with name.
+
+  Lines are expected to look like:
+
+     name: space separated values
+
+  Args:
+    name: string, parameter name.
+    lines: iterable of string, lines in the file.
+
+  Returns:
+    List of values in the lines that match.
+  """
+  retval = []
+  matches = itertools.ifilter(lambda x: x.startswith(name + ':'), lines)
+  for line in matches:
+    retval.extend(line[len(name)+1:].split())
+  return retval
+
+
+def wiki_escape(s):
+  """Detect WikiSyntax (i.e. InterCaps, a.k.a. CamelCase) and escape it."""
+  ret = []
+  for word in s.split():
+    if re.match(r'[A-Z]+[a-z]+[A-Z]', word):
+      word = '!%s' % word
+    ret.append(word)
+  return ' '.join(ret)
+
+
+def context_from_sample(api, keywords, dirname, desc):
+  """Return info for expanding a sample into a template.
+
+  Args:
+    api: string, name of api.
+    keywords: list of string, list of keywords for the given api.
+    dirname: string, directory name of the sample.
+    desc: string, long description of the sample.
+
+  Returns:
+    A dictionary of values useful for template expansion.
+  """
+  if api is None:
+    return None
+  else:
+    entry = DIRECTORY[api]
+    context = {
+        'api': api,
+        'version': entry['version'],
+        'api_name': wiki_escape(entry.get('title', entry.get('description'))),
+        'api_desc': wiki_escape(entry['description']),
+        'api_icon': entry['icons']['x32'],
+        'keywords': keywords,
+        'dir': dirname,
+        'dir_escaped': dirname.replace('/', '%2F'),
+        'desc': wiki_escape(desc),
+        }
+    return context
+
+
+def keyword_context_from_sample(keywords, dirname, desc):
+  """Return info for expanding a sample into a template.
+
+  Sample may not be about a specific sample.
+
+  Args:
+    keywords: list of string, list of keywords for the given api.
+    dirname: string, directory name of the sample.
+    desc: string, long description of the sample.
+
+  Returns:
+    A dictionary of values useful for template expansion.
+  """
+  context = {
+      'keywords': keywords,
+      'dir': dirname,
+      'dir_escaped': dirname.replace('/', '%2F'),
+      'desc': wiki_escape(desc),
+      }
+  return context
+
+
+def scan_readme_files(dirname):
+  """Scans all subdirs of dirname for README files.
+
+  Args:
+    dirname: string, name of directory to walk.
+
+  Returns:
+    (samples, keyword_set): list of information about all samples, the union
+      of all keywords found.
+  """
+  samples = []
+  keyword_set = set()
+
+  for root, dirs, files in os.walk(dirname):
+    if 'README' in files:
+      filename = os.path.join(root, 'README')
+      with open(filename, 'r') as f:
+        content = f.read()
+        lines = content.splitlines()
+        desc = ' '.join(itertools.takewhile(lambda x: x, lines))
+        api = get_lines('api', lines)
+        keywords = get_lines('keywords', lines)
+
+        for k in keywords:
+          if k not in KEYWORDS:
+            raise ValueError(
+                '%s is not a valid keyword in file %s' % (k, filename))
+        keyword_set.update(keywords)
+        if not api:
+          api = [None]
+        samples.append((api[0], keywords, root[1:], desc))
+
+  samples.sort()
+
+  return samples, keyword_set
+
+
+def main():
+  # Get all the information we need out of the README files in the samples.
+  samples, keyword_set = scan_readme_files('./samples')
+
+  # Now build a wiki page with all that information. Accumulate all the
+  # information as string to be concatenated when were done.
+  page = ['<wiki:toc max_depth="3" />\n= Samples By API =\n']
+
+  # All the samples, grouped by API.
+  current_api = None
+  for api, keywords, dirname, desc in samples:
+    context = context_from_sample(api, keywords, dirname, desc)
+    if context is None:
+      continue
+    if current_api != api:
+      page.append("""
+=== %(api_icon)s %(api_name)s ===
+
+%(api_desc)s
+
+Documentation for the %(api_name)s in [http://api-python-client-doc.appspot.com/%(api)s/%(version)s  PyDoc] 
+
+""" % context)
+      current_api = api
+
+    page.append('|| [http://code.google.com/p/google-api-python-client/source/browse/#hg%(dir_escaped)s %(dir)s] || %(desc)s ||\n' % context)
+
+  # Now group the samples by keywords.
+  for keyword, keyword_name in KEYWORDS.iteritems():
+    if keyword not in keyword_set:
+      continue
+    page.append('\n= %s Samples =\n\n' % keyword_name)
+    page.append('<table border=1 cellspacing=0 cellpadding=8px>\n')
+    for _, keywords, dirname, desc in samples:
+      context = keyword_context_from_sample(keywords, dirname, desc)
+      if keyword not in keywords:
+        continue
+      page.append("""
+<tr>
+  <td>[http://code.google.com/p/google-api-python-client/source/browse/#hg%(dir_escaped)s %(dir)s] </td>
+  <td> %(desc)s </td>
+</tr>""" % context)
+    page.append('</table>\n')
+
+  print ''.join(page)
+
+
+if __name__ == '__main__':
+  main()

samples/adexchangebuyer/README

@@ -0,0 +1,4 @@
+Samples for working with the Ad Exchange Buyer API.
+
+api: adexchangebuyer
+keywords: cmdline

samples/adsense/README

@@ -0,0 +1,4 @@
+A collection of command-line samples for the AdSense Management API.
+
+api: adsense
+keywords: cmdline

samples/analytics/README

@@ -0,0 +1,4 @@
+Command-line samples for producting reports with the Analytics API.
+
+api: analytics
+keywords: cmdline

samples/api-python-client-doc/README

@@ -1,6 +1,5 @@
-This sample is the code that drives
+This sample is the code that drives http://api-python-client-doc.appspot.com/
+It is an application that serves up the Python help documentation for each API.
 
-   http://api-python-client-doc.appspot.com/
-
-It is an application that serves up the Python help documentation
-for each API.
+api: discovery
+keywords: appengine

samples/appengine/README

@@ -0,0 +1,5 @@
+Simple Google+ sample that demonstrates the people API. Demontrates
+using the OAuth 2.0 Decorator for Google App Engine applications.
+
+api: plus
+keywords: appengine oauth2

samples/appengine_with_robots/README

@@ -0,0 +1,5 @@
+Sample application that demonstrates how to use AppAssertionCredentials
+to access an API.
+
+api: urlshortener
+keywords: appengine oauth2

samples/audit/README

@@ -0,0 +1,4 @@
+Prints the activities for a domain using the Audit API.
+
+api: audit
+keywords: cmdline

samples/blogger/README

@@ -0,0 +1,4 @@
+Retrieve the list of blogs and their posts for a user.
+
+api: blogger
+keywords: cmdline

samples/customsearch/README

@@ -0,0 +1,4 @@
+Search from the command-line.
+
+api: customsearch
+keywords: cmdline

samples/dailymotion/README

@@ -0,0 +1,3 @@
+Demonstrates using oauth2client against the DailyMotion API.
+
+keywords: oauth2 appengine

samples/django_sample/README

@@ -0,0 +1,4 @@
+Sample app demonstrating using oauth2client and the Google+ API from Django.
+
+api: plus
+keywords: oauth2 django

samples/groupssettings/README

@@ -0,0 +1,4 @@
+Sample for the Groups Settings API.
+
+api: groupssettings
+keywords: cmdline

samples/gtaskqueue_sample/README

@@ -1,6 +1,9 @@
 This acts as a sample as well as commandline tool for accessing Google TaskQueue
 APIs.
 
+api: taskqueue
+keywords: cmdline
+
 Installation
 ============
 

samples/latitude/README

@@ -0,0 +1,4 @@
+Add a new location via the Latitude API.
+
+api: latitude
+keywords: cmdline

samples/moderator/README

@@ -0,0 +1,4 @@
+Create new Moderator series and topics via the Moderator API.
+
+api: moderator
+keywords: cmdline

samples/plus/README

@@ -0,0 +1,4 @@
+Loop over all a user's activities and print a short snippet.
+
+api: plus
+keywords: cmdline pagination

samples/prediction/README

@@ -1,5 +1,8 @@
 Before you can run the prediction sample prediction.py, you must load some csv
-formatted data into Google Storage. You can do this by running setup.sh with a 
-bucket/object name of your choice. You must first create the bucket you want to 
-use. This can be done with the gsutil function or via the web UI (Storage 
+formatted data into Google Storage. You can do this by running setup.sh with a
+bucket/object name of your choice. You must first create the bucket you want
+to use. This can be done with the gsutil function or via the web UI (Storage
 Access) in the Google APIs Console.
+
+api: prediction
+keywords: cmdline

samples/searchforshopping/README

@@ -0,0 +1,4 @@
+Samples demonstrating the query capabilities for the Search API for Shopping.
+
+api: shopping
+keywords: cmdline

samples/service_account/README

@@ -0,0 +1,4 @@
+Sample that demonstrates working with Service Accounts.
+
+api: tasks
+keywords: oauth2

samples/tasks_appengine/README

@@ -1,2 +1,5 @@
 Sample code for Getting Started with Tasks API on App Engine article.
 http://code.google.com/appengine/articles/python/getting_started_with_tasks_api.html
+
+api: tasks
+keywords: appengine

samples/threadqueue/README

@@ -0,0 +1,4 @@
+Demonstrates using threading and thread queues for handling a high volume of requests.
+
+api: moderator
+keywords: cmdline threading

samples/translate/README

@@ -0,0 +1,5 @@
+Simple sample for the translate API.
+
+api: translate
+keywords: cmdline
+

samples/tz/README

@@ -4,6 +4,9 @@ based on the user's location, as determined by Google
 Latitude. To use this application you will need Google
 Latitude running on a mobile device.
 
+api: latitude
+keywords: cmdline
+
 Installation
 ============
   The google-api-python-client library will need to

samples/urlshortener/README

@@ -0,0 +1,4 @@
+Shortens and URL with the URL Shortener API.
+
+api: urlshortener
+keywords: cmdline