Merge "Generate plugins list through https requests"

data/devstack-plugins-registry.header

@@ -14,6 +14,6 @@ The following are plugins that a script has found in the openstack/
 namespace, which includes but is not limited to official OpenStack
 projects.
 
-+------------------+------------------------------------------------------------+------------+
-|Plugin Name       |URL                                                         |Date        |
-+------------------+------------------------------------------------------------+------------+
++------------------+-------------------------------------------------------------------------+
+|Plugin Name       |URL                                                                      |
++------------------+-------------------------------------------------------------------------+

tools/generate-devstack-plugins-list.py

@@ -0,0 +1,59 @@
+#! /usr/bin/env python
+
+# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This script is intended to be run as part of a periodic proposal bot
+# job in OpenStack infrastructure.
+#
+# In order to function correctly, the environment in which the
+# script runs must have
+#   * network access to the review.openstack.org Gerrit API
+#     working directory
+#   * network access to https://git.openstack.org/cgit
+
+import json
+import requests
+
+url = 'https://review.openstack.org/projects/'
+
+# This is what a project looks like
+'''
+  "openstack-attic/akanda": {
+    "id": "openstack-attic%2Fakanda",
+    "state": "READ_ONLY"
+  },
+'''
+
+def is_in_openstack_namespace(proj):
+    return proj.startswith('openstack/')
+
+# Rather than returning a 404 for a nonexistent file, cgit delivers a
+# 0-byte response to a GET request.  It also does not provide a
+# Content-Length in a HEAD response, so the way we tell if a file exists
+# is to check the length of the entire GET response body.
+def has_devstack_plugin(proj):
+    r = requests.get("https://git.openstack.org/cgit/%s/plain/devstack/plugin.sh" % proj)
+    if len(r.text) > 0:
+        return True
+    else:
+        False
+
+r = requests.get(url)
+projects = sorted(filter(is_in_openstack_namespace, json.loads(r.text[4:])))
+
+found_plugins = filter(has_devstack_plugin, projects)
+
+for project in found_plugins:
+    print project[10:]

tools/generate-devstack-plugins-list.sh

@@ -19,11 +19,19 @@
 #
 # In order to function correctly, the environment in which the
 # script runs must have
+#   * a writable doc/source directory relative to the current
+#     working directory
+#   AND ( (
 #   * git
 #   * all git repos meant to be searched for plugins cloned and
 #     at the desired level of up-to-datedness
-#   * a writable doc/source directory relative to the current
+#   * the environment variable git_dir pointing to the location
+#   * of said git repositories
+#   ) OR (
+#   * network access to the review.openstack.org Gerrit API
 #     working directory
+#   * network access to https://git.openstack.org/cgit
+#   ))
 #
 # If a file named data/devstack-plugins-registry.header or
 # data/devstack-plugins-registry.footer is found relative to the
@@ -35,25 +43,38 @@ declare -A plugins
 
 test -r data/devstack-plugins-registry.header && cat data/devstack-plugins-registry.header
 
-pushd ${git_dir:-/opt/openstack} >/dev/null
-for i in *; do
-    pushd ${i} >/dev/null
-    if output="$(git log --diff-filter=A --format='%cd' --date=short -1 -- devstack/plugin.sh)"; then
-        test -n "$output" && plugins[$i]=${output}
-    fi
+if test -n "$git_dir"; then
+    pushd ${git_dir} >/dev/null
+    for i in *; do
+        pushd ${i} >/dev/null
+        if output="$(git log --diff-filter=A --format='%cd' --date=short -1 -- devstack/plugin.sh)"; then
+            test -n "$output" && plugins[$i]=${output}
+        fi
+        popd >/dev/null
+    done
     popd >/dev/null
-done
-popd >/dev/null
 
-sorted_plugins=( $(for k in "${!plugins[@]}"; do echo "$k"; done | sort))
+    sorted_plugins=( $(for k in "${!plugins[@]}"; do echo "$k"; done | sort))
 
-for k in "${sorted_plugins[@]}"; do
-    project=${k:0:18}
-    giturl="git://git.openstack.org/openstack/${k:0:26}"
-    pdate="${plugins[$k]}"
-    printf "|%-18s|%-60s|%-12s|\n" "${project}" "${giturl}" "${pdate}"
-    printf "+------------------+------------------------------------------------------------+------------+\n"
-done
+    for k in "${sorted_plugins[@]}"; do
+        project=${k:0:18}
+        giturl="git://git.openstack.org/openstack/${k:0:26}"
+        pdate="${plugins[$k]}"
+        printf "|%-18s|%-60s (%-10s)|\n" "${project}" "${giturl}" "${pdate}"
+        printf "+------------------+-------------------------------------------------------------------------+\n"
+    done
+
+else
+    sorted_plugins=$(python tools/generate-devstack-plugins-list.py)
+
+    for k in ${sorted_plugins}; do
+        project=${k:0:18}
+        giturl="git://git.openstack.org/openstack/${k:0:26}"
+        printf "|%-18s|%-73s|\n" "${project}" "${giturl}"
+        printf "+------------------+-------------------------------------------------------------------------+\n"
+    done
+
+fi
 
 test -r data/devstack-plugins-registry.footer && cat data/devstack-plugins-registry.footer
 ) > doc/source/plugin-registry.rst