openstack/project-config
Code Issues Proposed changes

Merge "check that osc plugins do not break openstackclient"

Browse Source
This commit is contained in:
Jenkins
2016-01-05 09:36:45 +00:00
committed by Gerrit Code Review
parent f3b14c05c7 7159126bcd
commit 00a7bb32aa
4 changed files with 230 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
jenkins/jobs/osc.yaml
View File

@@ -39,3 +39,17 @@
- test-results
- devstack-logs
- console-log
- job:
name: check-osc-plugins
node: bare-trusty
builders:
- revoke-sudo
- link-logs
- net-info
- gerrit-git-prep
- shell: /usr/local/jenkins/slave_scripts/check-osc-plugins.sh
publishers:
- console-log

62
jenkins/scripts/check-osc-plugins.sh Executable file
View File

@@ -0,0 +1,62 @@
#!/bin/bash -xe
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Install all known OpenStackClient projects that have plugins. Then install
# the proposed change to see if there are any conflicting commands.
# install openstackclient plugins from source to catch conflicts earlier
function install_from_source {
repo=$1
root=$(mktemp -d)
$zc --cache-dir /opt/git --workspace ${root} \
git://git.openstack.org openstack/${repo}
(cd ${root}/openstack/${repo} && $venv/bin/pip install .)
rm -rf $root
}
zc='/usr/zuul-env/bin/zuul-cloner'
# setup a virtual environment to install all the plugins
venv_name='osc_plugins'
virtualenv $venv_name
trap "rm -rf $VENV" EXIT
venv=$(pwd)/$venv_name
# install known OpenStackClient plugins
install_from_source python-openstackclient
install_from_source python-barbicanclient
install_from_source python-congressclient
install_from_source python-cueclient
install_from_source python-designateclient
install_from_source python-heatclient
install_from_source python-ironicclient
install_from_source python-saharaclient
install_from_source python-tuskarclient
install_from_source python-zaqarclient
echo "Begin freeze output from $venv virtualenv:"
echo "======================================================================"
$venv/bin/pip freeze
echo "======================================================================"
# now check the current proposed change doesn't cause a conflict
# we should already be in the project's root directory where setup.py exists
echo "Installing the proposed change in directory: $(pwd)"
$venv/bin/pip install -e .
echo "Testing development version of openstack client, version:"
$venv/bin/openstack --version
# run the python script to actually check the commands now that we're setup
$venv/bin/python /usr/local/jenkins/slave_scripts/check_osc_commands.py

145
jenkins/scripts/check_osc_commands.py Executable file
View File

@@ -0,0 +1,145 @@
#! /usr/bin/env python
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
This module will use `pkg_resources` to scan commands for all OpenStackClient
plugins with the purpose of detecting duplicate commands.
"""
import pkg_resources
def find_duplicates():
"""Find duplicates commands.
Here we use `pkg_resources` to find all modules. There will be many modules
on a system, so we filter them out based on "openstack" since that is the
prefix that OpenStackClient plugins will have.
Each module has various entry points, each OpenStackClient command will
have an entrypoint. Each entry point has a short name (ep.name) which
is the command the user types, as well as a long name (ep.module_name)
which indicates from which module the entry point is from.
For example, the entry point and module for v3 user list is::
module => openstackclient.identity.v3
ep.name => user_list
ep.module_name => openstackclient.identity.v3.user
We keep a running tally of valid commands, duplicate commands and commands
that failed to load.
The resultant data structure for valid commands should look like::
{'user_list':
['openstackclient.identity.v3.user',
'openstackclient.identity.v2.0.user']
'flavor_list':
[openstackclient.compute.v2.flavor']
}
The same can be said for the duplicate and failed commands.
"""
valid_cmds = {}
duplicate_cmds = {}
failed_cmds = {}
# find all modules on the system
modules = set()
for dist in pkg_resources.working_set:
entry_map = pkg_resources.get_entry_map(dist)
modules.update(set(entry_map.keys()))
for module in modules:
# OpenStackClient plugins are prefixed with "openstack", skip otherwise
if not module.startswith('openstack'):
continue
# Iterate over all entry points
for ep in pkg_resources.iter_entry_points(module):
# cliff does a mapping between spaces and underscores
ep_name = ep.name.replace(' ', '_')
try:
ep.load()
except Exception:
failed_cmds.setdefault(ep_name, []).append(ep.module_name)
if _is_valid_command(ep_name, ep.module_name, valid_cmds):
valid_cmds.setdefault(ep_name, []).append(ep.module_name)
else:
duplicate_cmds.setdefault(ep_name, []).append(ep.module_name)
if duplicate_cmds:
print("Duplicate commands found...")
print(duplicate_cmds)
return True
if failed_cmds:
print("Some commands failed to load...")
print(failed_cmds)
return True
# Safely return False here with the full set of commands
print("Final set of commands...")
print(valid_cmds)
print("Found no duplicate commands, OK to merge!")
return False
def _is_valid_command(ep_name, ep_module_name, valid_cmds):
"""Determine if the entry point is valid.
Aside from a simple check to see if the entry point short name is in our
tally, we also need to check for allowed duplicates. For instance, in the
case of supporting multiple versions, then we want to allow for duplicate
commands. Both the identity v2 and v3 APIs support `user_list`, so these
are fine.
In order to determine if an entry point is a true duplicate we can check to
see if the module name roughly matches the module name of the entry point
that was initially added to the set of valid commands.
The following should trigger a match::
openstackclient.identity.v3.user and openstackclient.identity.v*.user
Whereas, the following should fail::
openstackclient.identity.v3.user and openstackclient.baremetal.v3.user
"""
if ep_name not in valid_cmds:
return True
else:
# there already exists an entry in the dictionary for the command...
module_parts = ep_module_name.split(".")
for valid_module_name in valid_cmds[ep_name]:
valid_module_parts = valid_module_name.split(".")
if (module_parts[0] == valid_module_parts[0] and
module_parts[1] == valid_module_parts[1] and
module_parts[3] == valid_module_parts[3]):
return True
return False
if __name__ == '__main__':
print("Checking 'openstack' plug-ins")
if find_duplicates():
exit(1)
else:
exit(0)

9
zuul/layout.yaml
View File

@@ -387,6 +387,10 @@ project-templates:
gate:
- 'gate-{name}-python34-constraints'
- name: osc-plugin-jobs
experimental:
- 'check-osc-plugins'
- name: pypy-jobs
check:
- 'gate-{name}-pypy'
@@ -1518,6 +1522,10 @@ jobs:
- name: keystone-propose-config-updates
branch: ^(?!stable/kilo).*$
- name: check-osc-plugins
branch: ^(?!stable/(kilo|liberty)).*$
voting: false
- name: gate-keystonemiddleware-tox-bandit
branch: ^(?!stable/kilo).*$
@@ -8493,6 +8501,7 @@ projects:
- name: translation-jobs
- name: release-notes-jobs
- name: lib-forward-testing
- name: osc-plugin-jobs
check:
- gate-osc-dsvm-functional
gate: