Prepare for dynamic generation of tempest plugin registry

This is based on the devstack plugin detection code, modified
to check setup.cfg for a tempest_plugins string.  It is to be
executed from a periodic proposal job, generating a list of
plugins in the same format.

Change-Id: I53d082319b6af6fb47083ea42c3ff6b109882e72

data/tempest-plugins-registry.header

@@ -0,0 +1,23 @@
+..
+  Note to patch submitters: this file is covered by a periodic proposal
+  job.  You should edit the files data/tempest-plugins-registry.footer
+  and data/tempest-plugins-registry.header instead of this one.
+
+==========================
+ Tempest Plugin Registry
+==========================
+
+Since we've created the external plugin mechanism, it's gotten used by
+a lot of projects. The following is a list of plugins that currently
+exist.
+
+Detected Plugins
+================
+
+The following are plugins that a script has found in the openstack/
+namespace, which includes but is not limited to official OpenStack
+projects.
+
++----------------------------+-------------------------------------------------------------------------+
+|Plugin Name                 |URL                                                                      |
++----------------------------+-------------------------------------------------------------------------+

doc/source/index.rst

@@ -11,6 +11,7 @@ Contents:
    HACKING
    REVIEWING
    plugin
+   plugin-registry
    library
 
 ------------

doc/source/plugin-registry.rst

@@ -0,0 +1,23 @@
+..
+  Note to patch submitters: this file is covered by a periodic proposal
+  job.  You should edit the files data/tempest-plugins-registry.footer
+  data/tempest-plugins-registry.header instead of this one.
+
+==========================
+ Tempest Plugin Registry
+==========================
+
+Since we've created the external plugin mechanism, it's gotten used by
+a lot of projects. The following is a list of plugins that currently
+exist.
+
+Detected Plugins
+================
+
+The following will list plugins that a script has found in the openstack/
+namespace, which includes but is not limited to official OpenStack
+projects.
+
++----------------------------+-------------------------------------------------------------------------+
+|Plugin Name                 |URL                                                                      |
++----------------------------+-------------------------------------------------------------------------+

tools/generate-tempest-plugins-list.py

@@ -0,0 +1,70 @@
+#! /usr/bin/env python
+
+# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This script is intended to be run as part of a periodic proposal bot
+# job in OpenStack infrastructure.
+#
+# In order to function correctly, the environment in which the
+# script runs must have
+#   * network access to the review.openstack.org Gerrit API
+#     working directory
+#   * network access to https://git.openstack.org/cgit
+
+import json
+import re
+import requests
+
+url = 'https://review.openstack.org/projects/'
+
+# This is what a project looks like
+'''
+  "openstack-attic/akanda": {
+    "id": "openstack-attic%2Fakanda",
+    "state": "READ_ONLY"
+  },
+'''
+
+
+def is_in_openstack_namespace(proj):
+    return proj.startswith('openstack/')
+
+# Rather than returning a 404 for a nonexistent file, cgit delivers a
+# 0-byte response to a GET request.  It also does not provide a
+# Content-Length in a HEAD response, so the way we tell if a file exists
+# is to check the length of the entire GET response body.
+
+
+def has_tempest_plugin(proj):
+    r = requests.get(
+        "https://git.openstack.org/cgit/%s/plain/setup.cfg" % proj)
+    p = re.compile('^tempest\.test_plugins', re.M)
+    if p.findall(r.text):
+        return True
+    else:
+        False
+
+r = requests.get(url)
+# Gerrit prepends 4 garbage octets to the JSON, in order to counter
+# cross-site scripting attacks.  Therefore we must discard it so the
+# json library won't choke.
+projects = sorted(filter(is_in_openstack_namespace, json.loads(r.text[4:])))
+
+found_plugins = filter(has_tempest_plugin, projects)
+
+# Every element of the found_plugins list begins with "openstack/".
+# We drop those initial 10 octets when printing the list.
+for project in found_plugins:
+    print(project[10:])

tools/generate-tempest-plugins-list.sh

@@ -0,0 +1,64 @@
+#!/bin/bash -ex
+
+# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This script is intended to be run as a periodic proposal bot job
+# in OpenStack infrastructure, though you can run it as a one-off.
+#
+# In order to function correctly, the environment in which the
+# script runs must have
+#   * a writable doc/source directory relative to the current
+#     working directory
+#   AND ( (
+#   * git
+#   * all git repos meant to be searched for plugins cloned and
+#     at the desired level of up-to-datedness
+#   * the environment variable git_dir pointing to the location
+#   * of said git repositories
+#   ) OR (
+#   * network access to the review.openstack.org Gerrit API
+#     working directory
+#   * network access to https://git.openstack.org/cgit
+#   ))
+#
+# If a file named data/tempest-plugins-registry.header or
+# data/tempest-plugins-registry.footer is found relative to the
+# current working directory, it will be prepended or appended to
+# the generated reStructuredText plugins table respectively.
+
+(
+declare -A plugins
+
+if [[ -r data/tempest-plugins-registry.header ]]; then
+    cat data/tempest-plugins-registry.header
+fi
+
+sorted_plugins=$(python tools/generate-tempest-plugins-list.py)
+
+for k in ${sorted_plugins}; do
+    project=${k:0:28}
+    giturl="git://git.openstack.org/openstack/${k:0:26}"
+    printf "|%-28s|%-73s|\n" "${project}" "${giturl}"
+    printf "+----------------------------+-------------------------------------------------------------------------+\n"
+done
+
+if [[ -r data/tempest-plugins-registry.footer ]]; then
+    cat data/tempest-plugins-registry.footer
+fi
+) > doc/source/plugin-registry.rst
+
+if [[ -n ${1} ]]; then
+    cp doc/source/plugin-registry.rst ${1}/doc/source/plugin-registry.rst
+fi