From a413652c250677bc881c51c4a592bacfa3faac69 Mon Sep 17 00:00:00 2001 From: Clint Adams Date: Thu, 10 Mar 2016 20:24:34 -0500 Subject: [PATCH] Prepare for dynamic generation of tempest plugin registry This is based on the devstack plugin detection code, modified to check setup.cfg for a tempest_plugins string. It is to be executed from a periodic proposal job, generating a list of plugins in the same format. Change-Id: I53d082319b6af6fb47083ea42c3ff6b109882e72 --- data/tempest-plugins-registry.header | 23 +++++++++ doc/source/index.rst | 1 + doc/source/plugin-registry.rst | 23 +++++++++ tools/generate-tempest-plugins-list.py | 70 ++++++++++++++++++++++++++ tools/generate-tempest-plugins-list.sh | 64 +++++++++++++++++++++++ 5 files changed, 181 insertions(+) create mode 100644 data/tempest-plugins-registry.header create mode 100644 doc/source/plugin-registry.rst create mode 100644 tools/generate-tempest-plugins-list.py create mode 100755 tools/generate-tempest-plugins-list.sh diff --git a/data/tempest-plugins-registry.header b/data/tempest-plugins-registry.header new file mode 100644 index 0000000000..9821e8e704 --- /dev/null +++ b/data/tempest-plugins-registry.header @@ -0,0 +1,23 @@ +.. + Note to patch submitters: this file is covered by a periodic proposal + job. You should edit the files data/tempest-plugins-registry.footer + and data/tempest-plugins-registry.header instead of this one. + +========================== + Tempest Plugin Registry +========================== + +Since we've created the external plugin mechanism, it's gotten used by +a lot of projects. The following is a list of plugins that currently +exist. + +Detected Plugins +================ + +The following are plugins that a script has found in the openstack/ +namespace, which includes but is not limited to official OpenStack +projects. + ++----------------------------+-------------------------------------------------------------------------+ +|Plugin Name |URL | ++----------------------------+-------------------------------------------------------------------------+ diff --git a/doc/source/index.rst b/doc/source/index.rst index 17def1c84f..f5de309a7c 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -11,6 +11,7 @@ Contents: HACKING REVIEWING plugin + plugin-registry library ------------ diff --git a/doc/source/plugin-registry.rst b/doc/source/plugin-registry.rst new file mode 100644 index 0000000000..517e5b8a96 --- /dev/null +++ b/doc/source/plugin-registry.rst @@ -0,0 +1,23 @@ +.. + Note to patch submitters: this file is covered by a periodic proposal + job. You should edit the files data/tempest-plugins-registry.footer + data/tempest-plugins-registry.header instead of this one. + +========================== + Tempest Plugin Registry +========================== + +Since we've created the external plugin mechanism, it's gotten used by +a lot of projects. The following is a list of plugins that currently +exist. + +Detected Plugins +================ + +The following will list plugins that a script has found in the openstack/ +namespace, which includes but is not limited to official OpenStack +projects. + ++----------------------------+-------------------------------------------------------------------------+ +|Plugin Name |URL | ++----------------------------+-------------------------------------------------------------------------+ diff --git a/tools/generate-tempest-plugins-list.py b/tools/generate-tempest-plugins-list.py new file mode 100644 index 0000000000..03dbd9b728 --- /dev/null +++ b/tools/generate-tempest-plugins-list.py @@ -0,0 +1,70 @@ +#! /usr/bin/env python + +# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script is intended to be run as part of a periodic proposal bot +# job in OpenStack infrastructure. +# +# In order to function correctly, the environment in which the +# script runs must have +# * network access to the review.openstack.org Gerrit API +# working directory +# * network access to https://git.openstack.org/cgit + +import json +import re +import requests + +url = 'https://review.openstack.org/projects/' + +# This is what a project looks like +''' + "openstack-attic/akanda": { + "id": "openstack-attic%2Fakanda", + "state": "READ_ONLY" + }, +''' + + +def is_in_openstack_namespace(proj): + return proj.startswith('openstack/') + +# Rather than returning a 404 for a nonexistent file, cgit delivers a +# 0-byte response to a GET request. It also does not provide a +# Content-Length in a HEAD response, so the way we tell if a file exists +# is to check the length of the entire GET response body. + + +def has_tempest_plugin(proj): + r = requests.get( + "https://git.openstack.org/cgit/%s/plain/setup.cfg" % proj) + p = re.compile('^tempest\.test_plugins', re.M) + if p.findall(r.text): + return True + else: + False + +r = requests.get(url) +# Gerrit prepends 4 garbage octets to the JSON, in order to counter +# cross-site scripting attacks. Therefore we must discard it so the +# json library won't choke. +projects = sorted(filter(is_in_openstack_namespace, json.loads(r.text[4:]))) + +found_plugins = filter(has_tempest_plugin, projects) + +# Every element of the found_plugins list begins with "openstack/". +# We drop those initial 10 octets when printing the list. +for project in found_plugins: + print(project[10:]) diff --git a/tools/generate-tempest-plugins-list.sh b/tools/generate-tempest-plugins-list.sh new file mode 100755 index 0000000000..ecff508c94 --- /dev/null +++ b/tools/generate-tempest-plugins-list.sh @@ -0,0 +1,64 @@ +#!/bin/bash -ex + +# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script is intended to be run as a periodic proposal bot job +# in OpenStack infrastructure, though you can run it as a one-off. +# +# In order to function correctly, the environment in which the +# script runs must have +# * a writable doc/source directory relative to the current +# working directory +# AND ( ( +# * git +# * all git repos meant to be searched for plugins cloned and +# at the desired level of up-to-datedness +# * the environment variable git_dir pointing to the location +# * of said git repositories +# ) OR ( +# * network access to the review.openstack.org Gerrit API +# working directory +# * network access to https://git.openstack.org/cgit +# )) +# +# If a file named data/tempest-plugins-registry.header or +# data/tempest-plugins-registry.footer is found relative to the +# current working directory, it will be prepended or appended to +# the generated reStructuredText plugins table respectively. + +( +declare -A plugins + +if [[ -r data/tempest-plugins-registry.header ]]; then + cat data/tempest-plugins-registry.header +fi + +sorted_plugins=$(python tools/generate-tempest-plugins-list.py) + +for k in ${sorted_plugins}; do + project=${k:0:28} + giturl="git://git.openstack.org/openstack/${k:0:26}" + printf "|%-28s|%-73s|\n" "${project}" "${giturl}" + printf "+----------------------------+-------------------------------------------------------------------------+\n" +done + +if [[ -r data/tempest-plugins-registry.footer ]]; then + cat data/tempest-plugins-registry.footer +fi +) > doc/source/plugin-registry.rst + +if [[ -n ${1} ]]; then + cp doc/source/plugin-registry.rst ${1}/doc/source/plugin-registry.rst +fi