From 7518301120c2a2036af494daab05be383f03d2c9 Mon Sep 17 00:00:00 2001
From: Masayuki Igawa <masayuki@igawa.io>
Date: Tue, 16 Jan 2018 14:30:44 +0900
Subject: [PATCH] Generate plugin list for registry document

This commit makes to generate a plugin list for the registry
document and, also fixes a python3 incompatibility and docs warnings
in generate-grenade-plugins-list.sh.
And this commit also removes doc/source/plugin-registry.rst because
it's generated by the script.

Change-Id: I391ca452f2e99e899ded8e4bcb5649a4511a7696
---
 .gitignore                             |  1 +
 data/grenade-plugins-registry.header   |  6 ++---
 doc/source/conf.py                     | 12 ++++++++-
 doc/source/plugin-registry.rst         | 24 ------------------
 tools/generate-grenade-plugins-list.py | 34 +++++++++++++++-----------
 tools/generate-grenade-plugins-list.sh |  8 +++---
 tox.ini                                |  1 +
 7 files changed, 40 insertions(+), 46 deletions(-)
 delete mode 100644 doc/source/plugin-registry.rst

diff --git a/.gitignore b/.gitignore
index c57f09af..900912fc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@ localrc
 *.swp
 doc/files
 doc/build
+doc/source/plugin-registry.rst
 shocco
 .tox
 AUTHORS
diff --git a/data/grenade-plugins-registry.header b/data/grenade-plugins-registry.header
index a35fb935..40117362 100644
--- a/data/grenade-plugins-registry.header
+++ b/data/grenade-plugins-registry.header
@@ -19,6 +19,6 @@ The following are plugins that a script has found in the openstack/
 namespace, which includes but is not limited to official OpenStack
 projects.
 
-+----------------------------+-------------------------------------------------------------------------+
-|Plugin Name                 |URL                                                                      |
-+----------------------------+-------------------------------------------------------------------------+
++----------------------------------------+-------------------------------------------------------------------------+
+|Plugin Name                             |URL                                                                      |
++----------------------------------------+-------------------------------------------------------------------------+
diff --git a/doc/source/conf.py b/doc/source/conf.py
index 259e3362..81607680 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -11,8 +11,18 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys
 import os
+import subprocess
+
+# Build the plugin registry
+def build_plugin_registry(app):
+    root_dir = os.path.dirname(
+        os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    subprocess.call(['tools/generate-grenade-plugins-list.sh'], cwd=root_dir)
+
+def setup(app):
+    if os.getenv('GENERATE_GRENADE_PLUGIN_LIST', 'true').lower() == 'true':
+        app.connect('builder-inited', build_plugin_registry)
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
diff --git a/doc/source/plugin-registry.rst b/doc/source/plugin-registry.rst
deleted file mode 100644
index cac47eeb..00000000
--- a/doc/source/plugin-registry.rst
+++ /dev/null
@@ -1,24 +0,0 @@
-..
-  Note to patch submitters: this file is about to be covered by a
-  periodic proposal job.  You should edit the files
-  data/grenade-plugins-registry.footer and
-  data/grenade-plugins-registry.header instead of this one.
-
-=========================
- Grenade Plugin Registry
-=========================
-
-Since we've created the external plugin mechanism, it's gotten used by
-a lot of projects. The following will be a list of plugins that currently
-exist.
-
-Detected Plugins
-================
-
-The following are plugins that a script has found in the openstack/
-namespace, which includes but is not limited to official OpenStack
-projects.
-
-+----------------------------+-------------------------------------------------------------------------+
-|Plugin Name                 |URL                                                                      |
-+----------------------------+-------------------------------------------------------------------------+
diff --git a/tools/generate-grenade-plugins-list.py b/tools/generate-grenade-plugins-list.py
index 84f6da95..714f61a4 100644
--- a/tools/generate-grenade-plugins-list.py
+++ b/tools/generate-grenade-plugins-list.py
@@ -24,7 +24,14 @@
 #   * network access to https://git.openstack.org/cgit
 
 import json
-import requests
+try:
+    # For Python 3.0 and later
+    from urllib.error import HTTPError
+    import urllib.request as urllib
+except ImportError:
+    # Fall back to Python 2's urllib2
+    import urllib2 as urllib
+    from urllib2 import HTTPError
 
 url = 'https://review.openstack.org/projects/'
 
@@ -39,21 +46,20 @@ url = 'https://review.openstack.org/projects/'
 def is_in_openstack_namespace(proj):
     return proj.startswith('openstack/')
 
-# Rather than returning a 404 for a nonexistent file, cgit delivers a
-# 0-byte response to a GET request.  It also does not provide a
-# Content-Length in a HEAD response, so the way we tell if a file exists
-# is to check the length of the entire GET response body.
-def has_grenade_plugin(proj):
-    r = requests.get("https://git.openstack.org/cgit/%s/plain/devstack/upgrade/upgrade.sh" % proj)
-    if len(r.text) > 0:
-        return True
-    else:
-        False
 
-r = requests.get(url)
-projects = sorted(filter(is_in_openstack_namespace, json.loads(r.text[4:])))
+def has_grenade_plugin(proj):
+    try:
+        r = urllib.urlopen(
+            "https://git.openstack.org/cgit/%s/plain/devstack/upgrade/upgrade.sh" % proj)
+        return True
+    except HTTPError as err:
+        if err.code == 404:
+            return False
+
+r = urllib.urlopen(url)
+projects = sorted(filter(is_in_openstack_namespace, json.loads(r.read()[4:])))
 
 found_plugins = filter(has_grenade_plugin, projects)
 
 for project in found_plugins:
-    print project[10:]
+    print(project[10:])
diff --git a/tools/generate-grenade-plugins-list.sh b/tools/generate-grenade-plugins-list.sh
index 49532f3a..b38d129a 100755
--- a/tools/generate-grenade-plugins-list.sh
+++ b/tools/generate-grenade-plugins-list.sh
@@ -48,10 +48,10 @@ fi
 sorted_plugins=$(python tools/generate-grenade-plugins-list.py)
 
 for k in ${sorted_plugins}; do
-    project=${k:0:28}
-    giturl="git://git.openstack.org/openstack/${k:0:26}"
-    printf "|%-28s|%-73s|\n" "${project}" "${giturl}"
-    printf "+----------------------------+-------------------------------------------------------------------------+\n"
+    project=${k:0:40}
+    giturl="git://git.openstack.org/openstack/${k:0:36}"
+    printf "|%-40s|%-73s|\n" "${project}" "${giturl}"
+    printf "+----------------------------------------+-------------------------------------------------------------------------+\n"
 done
 
 if [[ -r data/grenade-plugins-registry.footer ]]; then
diff --git a/tox.ini b/tox.ini
index b5eeae5b..01e27fc1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,6 +7,7 @@ skipsdist = True
 install_command = pip install -U {opts} {packages}
 setenv = VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/test-requirements.txt
+passenv = GENERATE_GRENADE_PLUGIN_LIST
 
 [testenv:venv]
 commands = {posargs}