Merge "Remove unused jenkins/scripts scripts"
This commit is contained in:
Zuul
Gerrit Code Review
commit
7ab9e5175e
@ -1,75 +0,0 @@
|
||||
#!/bin/bash -xe
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Install all known OpenStackClient projects that have plugins. Then install
|
||||
# the proposed change to see if there are any conflicting commands.
|
||||
|
||||
# install openstackclient plugins from source to catch conflicts earlier
|
||||
function install_from_source {
|
||||
repo=$1
|
||||
root=$(mktemp -d)
|
||||
$zc --cache-dir /opt/git --workspace ${root} \
|
||||
https://git.openstack.org openstack/${repo}
|
||||
(cd ${root}/openstack/${repo} && $venv/bin/pip install .)
|
||||
rm -rf $root
|
||||
}
|
||||
|
||||
zc='/usr/zuul-env/bin/zuul-cloner'
|
||||
|
||||
# setup a virtual environment to install all the plugins
|
||||
venv_name='osc_plugins'
|
||||
trap "rm -rf $venv_name" EXIT
|
||||
virtualenv $venv_name
|
||||
venv=$(pwd)/$venv_name
|
||||
|
||||
# install known OpenStackClient plugins
|
||||
install_from_source python-openstackclient
|
||||
install_from_source python-barbicanclient
|
||||
install_from_source python-cloudkittyclient
|
||||
install_from_source python-congressclient
|
||||
install_from_source python-designateclient
|
||||
install_from_source python-heatclient
|
||||
install_from_source python-ironicclient
|
||||
install_from_source python-ironic-inspector-client
|
||||
install_from_source python-karborclient
|
||||
install_from_source python-mistralclient
|
||||
install_from_source python-muranoclient
|
||||
install_from_source python-neutronclient
|
||||
install_from_source python-octaviaclient
|
||||
install_from_source python-pankoclient
|
||||
install_from_source python-rsdclient
|
||||
install_from_source python-saharaclient
|
||||
install_from_source python-searchlightclient
|
||||
install_from_source python-senlinclient
|
||||
install_from_source python-tripleoclient
|
||||
install_from_source python-troveclient
|
||||
install_from_source python-vitrageclient
|
||||
install_from_source python-watcherclient
|
||||
install_from_source python-zaqarclient
|
||||
|
||||
echo "Begin freeze output from $venv virtualenv:"
|
||||
echo "======================================================================"
|
||||
$venv/bin/pip freeze
|
||||
echo "======================================================================"
|
||||
|
||||
# now check the current proposed change doesn't cause a conflict
|
||||
# we should already be in the project's root directory where setup.py exists
|
||||
echo "Installing the proposed change in directory: $(pwd)"
|
||||
$venv/bin/pip install -e .
|
||||
|
||||
echo "Testing development version of openstack client, version:"
|
||||
$venv/bin/openstack --version
|
||||
|
||||
# run the python script to actually check the commands now that we're setup
|
||||
$venv/bin/python /usr/local/jenkins/slave_scripts/check_osc_commands.py
|
||||
@ -1,38 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import requests
|
||||
import requestsexceptions
|
||||
import yaml
|
||||
|
||||
|
||||
def main():
|
||||
requestsexceptions.squelch_warnings()
|
||||
data = yaml.load(open('openstack_catalog/web/static/assets.yaml'))
|
||||
|
||||
assets = {}
|
||||
for a in data['assets']:
|
||||
url = a.get('attributes', {}).get('url')
|
||||
if url:
|
||||
r = requests.head(url, allow_redirects=True)
|
||||
if r.status_code != 200:
|
||||
assets[a['name']] = {'active': False}
|
||||
|
||||
with open('openstack_catalog/web/static/assets_dead.yaml', 'w') as out:
|
||||
out.write(yaml.safe_dump({"assets": assets}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1,181 +0,0 @@
|
||||
#! /usr/bin/env python
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
This module will use `pkg_resources` to scan commands for all OpenStackClient
|
||||
plugins with the purpose of detecting duplicate commands.
|
||||
"""
|
||||
|
||||
import pkg_resources
|
||||
import traceback
|
||||
|
||||
|
||||
def find_duplicates():
|
||||
"""Find duplicates commands.
|
||||
|
||||
Here we use `pkg_resources` to find all modules. There will be many modules
|
||||
on a system, so we filter them out based on "openstack" since that is the
|
||||
prefix that OpenStackClient plugins will have.
|
||||
|
||||
Each module has various entry points, each OpenStackClient command will
|
||||
have an entrypoint. Each entry point has a short name (ep.name) which
|
||||
is the command the user types, as well as a long name (ep.module_name)
|
||||
which indicates from which module the entry point is from.
|
||||
|
||||
For example, the entry point and module for v3 user list is::
|
||||
|
||||
module => openstackclient.identity.v3
|
||||
ep.name => user_list
|
||||
ep.module_name => openstackclient.identity.v3.user
|
||||
|
||||
We keep a running tally of valid commands, duplicate commands and commands
|
||||
that failed to load.
|
||||
|
||||
The resultant data structure for valid commands should look like::
|
||||
|
||||
{'user_list':
|
||||
['openstackclient.identity.v3.user',
|
||||
'openstackclient.identity.v2.0.user']
|
||||
'flavor_list':
|
||||
[openstackclient.compute.v2.flavor']
|
||||
}
|
||||
|
||||
The same can be said for the duplicate and failed commands.
|
||||
"""
|
||||
|
||||
valid_cmds = {}
|
||||
duplicate_cmds = {}
|
||||
failed_cmds = {}
|
||||
|
||||
# find all modules on the system
|
||||
modules = set()
|
||||
for dist in pkg_resources.working_set:
|
||||
entry_map = pkg_resources.get_entry_map(dist)
|
||||
modules.update(set(entry_map.keys()))
|
||||
|
||||
for module in modules:
|
||||
# OpenStackClient plugins are prefixed with "openstack", skip otherwise
|
||||
if not module.startswith('openstack'):
|
||||
continue
|
||||
|
||||
# Iterate over all entry points
|
||||
for ep in pkg_resources.iter_entry_points(module):
|
||||
|
||||
# Check for a colon, since valid entrypoints will have one, for
|
||||
# example: quota_show = openstackclient.common.quota:ShowQuota
|
||||
# and plugin entrypoints will not, for
|
||||
# example: orchestration = heatclient.osc.plugin
|
||||
if ':' not in str(ep):
|
||||
continue
|
||||
|
||||
# cliff does a mapping between spaces and underscores
|
||||
ep_name = ep.name.replace(' ', '_')
|
||||
|
||||
try:
|
||||
ep.load()
|
||||
except Exception:
|
||||
exc_string = traceback.format_exc()
|
||||
message = "{}\n{}".format(ep.module_name, exc_string)
|
||||
failed_cmds.setdefault(ep_name, []).append(message)
|
||||
|
||||
if _is_valid_command(ep_name, ep.module_name, valid_cmds):
|
||||
valid_cmds.setdefault(ep_name, []).append(ep.module_name)
|
||||
else:
|
||||
duplicate_cmds.setdefault(ep_name, []).append(ep.module_name)
|
||||
|
||||
if duplicate_cmds:
|
||||
print("Duplicate commands found...")
|
||||
print(duplicate_cmds)
|
||||
return True
|
||||
if failed_cmds:
|
||||
print("Some commands failed to load...")
|
||||
print(failed_cmds)
|
||||
return True
|
||||
|
||||
overlap_cmds = _check_command_overlap(valid_cmds)
|
||||
if overlap_cmds:
|
||||
print("WARNING: Some commands overlap...")
|
||||
print(overlap_cmds)
|
||||
# FIXME(stevemar): when we determine why commands are overlapping
|
||||
# we can uncomment the line below.
|
||||
# return True
|
||||
|
||||
# Safely return False here with the full set of commands
|
||||
print("Final set of commands...")
|
||||
print(valid_cmds)
|
||||
print("Found no duplicate or overlapping commands, OK to merge!")
|
||||
return False
|
||||
|
||||
|
||||
def _check_command_overlap(valid_cmds):
|
||||
"""Determine if the entry point overlaps with another command.
|
||||
|
||||
For example, if one plugin creates the command "object1 action",
|
||||
and another plugin creates the command "object1 action object2",
|
||||
the object2 command is unreachable since it overlaps the
|
||||
namespace.
|
||||
"""
|
||||
overlap_cmds = {}
|
||||
for ep_name, ep_mods in valid_cmds.iteritems():
|
||||
# Skip openstack.cli.base client entry points
|
||||
for ep_mod in ep_mods:
|
||||
for ep_name_search in valid_cmds.keys():
|
||||
if ep_name_search.startswith(ep_name + "_"):
|
||||
overlap_cmds.setdefault(ep_name, []).append(ep_name_search)
|
||||
return overlap_cmds
|
||||
|
||||
|
||||
def _is_valid_command(ep_name, ep_module_name, valid_cmds):
|
||||
"""Determine if the entry point is valid.
|
||||
|
||||
Aside from a simple check to see if the entry point short name is in our
|
||||
tally, we also need to check for allowed duplicates. For instance, in the
|
||||
case of supporting multiple versions, then we want to allow for duplicate
|
||||
commands. Both the identity v2 and v3 APIs support `user_list`, so these
|
||||
are fine.
|
||||
|
||||
In order to determine if an entry point is a true duplicate we can check to
|
||||
see if the module name roughly matches the module name of the entry point
|
||||
that was initially added to the set of valid commands.
|
||||
|
||||
The following should trigger a match::
|
||||
|
||||
openstackclient.identity.v3.user and openstackclient.identity.v*.user
|
||||
|
||||
Whereas, the following should fail::
|
||||
|
||||
openstackclient.identity.v3.user and openstackclient.baremetal.v3.user
|
||||
|
||||
"""
|
||||
|
||||
if ep_name not in valid_cmds:
|
||||
return True
|
||||
else:
|
||||
# there already exists an entry in the dictionary for the command...
|
||||
module_parts = ep_module_name.split(".")
|
||||
for valid_module_name in valid_cmds[ep_name]:
|
||||
valid_module_parts = valid_module_name.split(".")
|
||||
if (module_parts[0] == valid_module_parts[0] and
|
||||
module_parts[1] == valid_module_parts[1] and
|
||||
module_parts[3] == valid_module_parts[3]):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("Checking 'openstack' plug-ins")
|
||||
if find_duplicates():
|
||||
exit(1)
|
||||
else:
|
||||
exit(0)
|
||||
@ -1,7 +0,0 @@
|
||||
#!/bin/bash -x
|
||||
lvremove -f /dev/main/last_root
|
||||
lvrename /dev/main/root last_root
|
||||
lvcreate -L20G -s -n root /dev/main/orig_root
|
||||
APPEND="$(cat /proc/cmdline)"
|
||||
kexec -l /vmlinuz --initrd=/initrd.img --append="$APPEND"
|
||||
nohup bash -c "sleep 2; kexec -e" </dev/null >/dev/null 2>&1 &
|
||||
@ -1,44 +0,0 @@
|
||||
#!/bin/bash -x
|
||||
#
|
||||
# Copyright 2013 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
# Upload java packages to maven repositories
|
||||
|
||||
PROJECT=$1
|
||||
VERSION=$2
|
||||
META_DATA_FILE=$3
|
||||
PLUGIN_FILE=$4
|
||||
|
||||
# Strip project name and extension leaving only the version.
|
||||
VERSION=$(echo ${PLUGIN_FILE} | sed -n "s/${PROJECT}-\(.*\).jar/\1/p")
|
||||
|
||||
# generate pom file with version info
|
||||
POM_IN_ZIP=$(unzip -Z -1 ${PLUGIN_FILE}|grep pom.xml)
|
||||
unzip -o -j ${PLUGIN_FILE} ${POM_IN_ZIP}
|
||||
sed "s/\${{project-version}}/${VERSION}/g" <pom.xml >${META_DATA_FILE}
|
||||
|
||||
# deploy plugin artifacts from workspace to maven central repository
|
||||
MAVEN_REPO="https://oss.sonatype.org/content/groups/public/maven"
|
||||
MAVEN_REPO_CREDS="/home/jenkins/.mavencentral-curl"
|
||||
|
||||
curl -X PUT \
|
||||
--config ${MAVEN_REPO_CREDS} \
|
||||
--data-binary @${META_DATA_FILE} \
|
||||
-i "${MAVEN_REPO}/${PROJECT}/${VERSION}/${META_DATA_FILE}" > /dev/null 2>&1
|
||||
|
||||
curl -X PUT \
|
||||
--config ${MAVEN_REPO_CREDS} \
|
||||
--data-binary @${PLUGIN_FILE} \
|
||||
-i "${MAVEN_REPO}/${PROJECT}/${VERSION}/${PLUGIN_FILE}" > /dev/null 2>&1
|
||||
@ -1,22 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import sys
|
||||
from subprocess import Popen, PIPE
|
||||
|
||||
p = Popen(["ping", sys.argv[1]], stdout=PIPE)
|
||||
while True:
|
||||
line = p.stdout.readline().strip()
|
||||
if 'bytes from' in line:
|
||||
p.terminate()
|
||||
sys.exit(0)
|
||||
@ -1,269 +0,0 @@
|
||||
#! /usr/bin/env python
|
||||
# Copyright (C) 2011 OpenStack, LLC.
|
||||
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
|
||||
# Copyright (c) 2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import contextlib
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
requirement = None
|
||||
project = None
|
||||
|
||||
|
||||
def run_command(cmd):
|
||||
print(cmd)
|
||||
cmd_list = shlex.split(str(cmd))
|
||||
p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
(out, err) = p.communicate()
|
||||
if p.returncode != 0:
|
||||
raise SystemError(err)
|
||||
return (out.strip(), err.strip())
|
||||
|
||||
|
||||
class RequirementsList(object):
|
||||
def __init__(self, name, project):
|
||||
self.name = name
|
||||
self.reqs_by_file = {}
|
||||
self.project = project
|
||||
self.failed = False
|
||||
|
||||
@property
|
||||
def reqs(self):
|
||||
return {k: v for d in self.reqs_by_file.values()
|
||||
for k, v in d.items()}
|
||||
|
||||
def extract_reqs(self, content, strict):
|
||||
reqs = collections.defaultdict(set)
|
||||
parsed = requirement.parse(content)
|
||||
for name, entries in parsed.items():
|
||||
if not name:
|
||||
# Comments and other unprocessed lines
|
||||
continue
|
||||
list_reqs = [r for (r, line) in entries]
|
||||
# Strip the comments out before checking if there are duplicates
|
||||
list_reqs_stripped = [r._replace(comment='') for r in list_reqs]
|
||||
if strict and len(list_reqs_stripped) != len(set(
|
||||
list_reqs_stripped)):
|
||||
print("Requirements file has duplicate entries "
|
||||
"for package %s : %r." % (name, list_reqs))
|
||||
self.failed = True
|
||||
reqs[name].update(list_reqs)
|
||||
return reqs
|
||||
|
||||
def process(self, strict=True):
|
||||
"""Convert the project into ready to use data.
|
||||
|
||||
- an iterable of requirement sets to check
|
||||
- each set has the following rules:
|
||||
- each has a list of Requirements objects
|
||||
- duplicates are not permitted within that list
|
||||
"""
|
||||
print("Checking %(name)s" % {'name': self.name})
|
||||
# First, parse.
|
||||
for fname, content in self.project.get('requirements', {}).items():
|
||||
print("Processing %(fname)s" % {'fname': fname})
|
||||
if strict and not content.endswith('\n'):
|
||||
print("Requirements file %s does not "
|
||||
"end with a newline." % fname)
|
||||
self.reqs_by_file[fname] = self.extract_reqs(content, strict)
|
||||
|
||||
for name, content in project.extras(self.project).items():
|
||||
print("Processing .[%(extra)s]" % {'extra': name})
|
||||
self.reqs_by_file[name] = self.extract_reqs(content, strict)
|
||||
|
||||
|
||||
def grab_args():
|
||||
"""Grab and return arguments"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Check if project requirements have changed"
|
||||
)
|
||||
parser.add_argument('--local', action='store_true',
|
||||
help='check local changes (not yet in git)')
|
||||
parser.add_argument('branch', nargs='?', default='master',
|
||||
help='target branch for diffs')
|
||||
parser.add_argument('--zc', help='what zuul cloner to call')
|
||||
parser.add_argument('--reqs', help='use a specified requirements tree')
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def tempdir():
|
||||
try:
|
||||
reqroot = tempfile.mkdtemp()
|
||||
yield reqroot
|
||||
finally:
|
||||
shutil.rmtree(reqroot)
|
||||
|
||||
|
||||
def install_and_load_requirements(reqroot, reqdir):
|
||||
sha = run_command("git --git-dir %s/.git rev-parse HEAD" % reqdir)[0]
|
||||
print("requirements git sha: %s" % sha)
|
||||
req_venv = os.path.join(reqroot, 'venv')
|
||||
req_pip = os.path.join(req_venv, 'bin/pip')
|
||||
req_lib = os.path.join(req_venv, 'lib/python2.7/site-packages')
|
||||
out, err = run_command("virtualenv " + req_venv)
|
||||
out, err = run_command(req_pip + " install " + reqdir)
|
||||
sys.path.append(req_lib)
|
||||
global project
|
||||
global requirement
|
||||
from openstack_requirements import project # noqa
|
||||
from openstack_requirements import requirement # noqa
|
||||
|
||||
|
||||
def _is_requirement_in_global_reqs(req, global_reqs):
|
||||
# Compare all fields except the extras field as the global
|
||||
# requirements should not have any lines with the extras syntax
|
||||
# example: oslo.db[xyz]<1.2.3
|
||||
for req2 in global_reqs:
|
||||
if (req.package == req2.package and
|
||||
req.location == req2.location and
|
||||
req.specifiers == req2.specifiers and
|
||||
req.markers == req2.markers and
|
||||
req.comment == req2.comment):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
args = grab_args()
|
||||
branch = args.branch
|
||||
failed = False
|
||||
|
||||
# build a list of requirements from the global list in the
|
||||
# openstack/requirements project so we can match them to the changes
|
||||
with tempdir() as reqroot:
|
||||
# Only clone requirements repo if no local repo is specified
|
||||
# on the command line.
|
||||
if args.reqs is None:
|
||||
reqdir = os.path.join(reqroot, "openstack/requirements")
|
||||
if args.zc is not None:
|
||||
zc = args.zc
|
||||
else:
|
||||
zc = '/usr/zuul-env/bin/zuul-cloner'
|
||||
out, err = run_command("%(zc)s "
|
||||
"--cache-dir /opt/git "
|
||||
"--workspace %(root)s "
|
||||
"https://git.openstack.org "
|
||||
"openstack/requirements"
|
||||
% dict(zc=zc, root=reqroot))
|
||||
print(out)
|
||||
print(err)
|
||||
else:
|
||||
reqdir = args.reqs
|
||||
|
||||
install_and_load_requirements(reqroot, reqdir)
|
||||
global_reqs = requirement.parse(
|
||||
open(reqdir + '/global-requirements.txt', 'rt').read())
|
||||
for k, entries in global_reqs.items():
|
||||
# Discard the lines: we don't need them.
|
||||
global_reqs[k] = set(r for (r, line) in entries)
|
||||
blacklist = requirement.parse(
|
||||
open(reqdir + '/blacklist.txt', 'rt').read())
|
||||
cwd = os.getcwd()
|
||||
# build a list of requirements in the proposed change,
|
||||
# and check them for style violations while doing so
|
||||
head = run_command("git rev-parse HEAD")[0]
|
||||
head_proj = project.read(cwd)
|
||||
head_reqs = RequirementsList('HEAD', head_proj)
|
||||
# Don't apply strict parsing rules to stable branches.
|
||||
# Reasoning is:
|
||||
# - devstack etc protect us from functional issues
|
||||
# - we're backporting to stable, so guarding against
|
||||
# aesthetics and DRY concerns is not our business anymore
|
||||
# - if in future we have other not-functional linty style
|
||||
# things to add, we don't want them to affect stable
|
||||
# either.
|
||||
head_strict = not branch.startswith('stable/')
|
||||
head_reqs.process(strict=head_strict)
|
||||
|
||||
if not args.local:
|
||||
# build a list of requirements already in the target branch,
|
||||
# so that we can create a diff and identify what's being changed
|
||||
run_command("git remote update")
|
||||
run_command("git checkout remotes/origin/%s" % branch)
|
||||
branch_proj = project.read(cwd)
|
||||
|
||||
# switch back to the proposed change now
|
||||
run_command("git checkout %s" % head)
|
||||
else:
|
||||
branch_proj = {'root': cwd}
|
||||
branch_reqs = RequirementsList(branch, branch_proj)
|
||||
# Don't error on the target branch being broken.
|
||||
branch_reqs.process(strict=False)
|
||||
|
||||
# iterate through the changing entries and see if they match the global
|
||||
# equivalents we want enforced
|
||||
for fname, freqs in head_reqs.reqs_by_file.items():
|
||||
print("Validating %(fname)s" % {'fname': fname})
|
||||
for name, reqs in freqs.items():
|
||||
counts = {}
|
||||
if (name in branch_reqs.reqs and
|
||||
reqs == branch_reqs.reqs[name]):
|
||||
# Unchanged [or a change that preserves a current value]
|
||||
continue
|
||||
if name in blacklist:
|
||||
# Blacklisted items are not synced and are managed
|
||||
# by project teams as they see fit, so no further
|
||||
# testing is needed.
|
||||
continue
|
||||
if name not in global_reqs:
|
||||
failed = True
|
||||
print("Requirement %s not in openstack/requirements" %
|
||||
str(reqs))
|
||||
continue
|
||||
if reqs == global_reqs[name]:
|
||||
continue
|
||||
for req in reqs:
|
||||
if req.extras:
|
||||
for extra in req.extras:
|
||||
counts[extra] = counts.get(extra, 0) + 1
|
||||
else:
|
||||
counts[''] = counts.get('', 0) + 1
|
||||
if not _is_requirement_in_global_reqs(
|
||||
req, global_reqs[name]):
|
||||
failed = True
|
||||
print("Requirement for package %s : %s does "
|
||||
"not match openstack/requirements value : %s" % (
|
||||
name, str(req), str(global_reqs[name])))
|
||||
for extra, count in counts.items():
|
||||
if count != len(global_reqs[name]):
|
||||
failed = True
|
||||
print("Package %s%s requirement does not match "
|
||||
"number of lines (%d) in "
|
||||
"openstack/requirements" % (
|
||||
name,
|
||||
('[%s]' % extra) if extra else '',
|
||||
len(global_reqs[name])))
|
||||
|
||||
# report the results
|
||||
if failed or head_reqs.failed or branch_reqs.failed:
|
||||
print("*** Incompatible requirement found!")
|
||||
print("*** See http://docs.openstack.org/developer/requirements")
|
||||
sys.exit(1)
|
||||
print("Updated requirements match openstack/requirements.")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1,36 +0,0 @@
|
||||
#!/bin/bash -xe
|
||||
#
|
||||
# Copyright 2012 Hewlett-Packard Development Company, L.P.
|
||||
# Copyright 2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
# Retrieve a python sdist and upload it to pypi with Curl.
|
||||
|
||||
PROJECT=$1
|
||||
TARBALL_SITE=$2
|
||||
TAG=$(echo $ZUUL_REF | sed 's/^refs.tags.//')
|
||||
|
||||
# Look in the setup.cfg to determine if a package name is specified, but
|
||||
# fall back on the project name if necessary
|
||||
DISTNAME=$(/usr/local/jenkins/slave_scripts/pypi-extract-name.py --tarball \
|
||||
|| echo $PROJECT)
|
||||
FILENAME="$DISTNAME-$TAG.tar.gz"
|
||||
|
||||
rm -rf *tar.gz
|
||||
curl --fail -o $FILENAME https://$TARBALL_SITE/$PROJECT/$FILENAME
|
||||
|
||||
# Make sure we actually got a gzipped file
|
||||
file -b $FILENAME | grep gzip
|
||||
|
||||
twine upload -r pypi $FILENAME
|
||||
@ -1,38 +0,0 @@
|
||||
#!/bin/bash -xe
|
||||
#
|
||||
# Copyright 2012 Hewlett-Packard Development Company, L.P.
|
||||
# Copyright 2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
# Retrieve supported python wheels and upload them to pypi with Curl.
|
||||
|
||||
PROJECT=$1
|
||||
TARBALL_SITE=$2
|
||||
TAG=$(echo $ZUUL_REF | sed 's/^refs.tags.//')
|
||||
|
||||
# Look in the setup.cfg to determine if a package name is specified, but
|
||||
# fall back on the project name if necessary
|
||||
DISTNAME=$(/usr/local/jenkins/slave_scripts/pypi-extract-name.py --wheel \
|
||||
|| echo $PROJECT)
|
||||
# Look in the setup.cfg to see if this is a universal wheel or not
|
||||
WHEELTYPE=$(/usr/local/jenkins/slave_scripts/pypi-extract-universal.py)
|
||||
FILENAME="$DISTNAME-$TAG-$WHEELTYPE-none-any.whl"
|
||||
|
||||
rm -rf *.whl
|
||||
curl --fail -o $FILENAME https://$TARBALL_SITE/$PROJECT/$FILENAME
|
||||
|
||||
# Make sure we actually got a wheel
|
||||
file -b $FILENAME | grep -i zip
|
||||
|
||||
twine upload -r pypi $FILENAME
|
||||
@ -1,26 +0,0 @@
|
||||
#!/bin/bash -xe
|
||||
|
||||
# Run coverage via tox. Also, run pbr freeze on the
|
||||
# resulting environment at the end so that we have a record of exactly
|
||||
# what packages we ended up testing.
|
||||
|
||||
export NOSE_COVER_HTML=1
|
||||
export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
|
||||
|
||||
venv=${1:-cover}
|
||||
|
||||
# Workaround the combo of tox running setup.py outside of virtualenv
|
||||
# and RHEL having an old distribute. The next line can be removed
|
||||
# when either get fixed.
|
||||
python setup.py --version
|
||||
|
||||
tox -e$venv
|
||||
result=$?
|
||||
[ -e .tox/$venv/bin/pbr ] && freezecmd=pbr || freezecmd=pip
|
||||
|
||||
echo "Begin $freezecmd freeze output from test virtualenv:"
|
||||
echo "======================================================================"
|
||||
.tox/${venv}/bin/${freezecmd} freeze
|
||||
echo "======================================================================"
|
||||
|
||||
exit $result
|
||||
@ -1,94 +0,0 @@
|
||||
#!/bin/bash -xe
|
||||
|
||||
# If a bundle file is present, call tox with the jenkins version of
|
||||
# the test environment so it is used. Otherwise, use the normal
|
||||
# (non-bundle) test environment. Also, run pbr freeze on the
|
||||
# resulting environment at the end so that we have a record of exactly
|
||||
# what packages we ended up testing.
|
||||
#
|
||||
|
||||
venv=${1:-venv}
|
||||
tags_handling=${2:both}
|
||||
|
||||
mkdir -p doc/build
|
||||
export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
|
||||
# The "python setup.py build_sphinx" is intentionally executed instead of
|
||||
# "tox -edocs", because it's the standard python project build interface
|
||||
# specified in OpenStack Project Testing Interface:
|
||||
# http://governance.openstack.org/reference/project-testing-interface.html
|
||||
tox -e$venv -- python setup.py build_sphinx
|
||||
result=$?
|
||||
|
||||
# If the build has not already failed and whereto is installed then
|
||||
# test the redirects defined in the project.
|
||||
if [ $result -eq 0 ]; then
|
||||
if [ -e .tox/$venv/bin/whereto ]; then
|
||||
tox -e $venv -- whereto doc/source/_extra/.htaccess doc/test/redirect-tests.txt
|
||||
result=$?
|
||||
fi
|
||||
fi
|
||||
|
||||
[ -e .tox/$venv/bin/pbr ] && freezecmd=pbr || freezecmd=pip
|
||||
|
||||
echo "Begin pbr freeze output from test virtualenv:"
|
||||
echo "======================================================================"
|
||||
.tox/${venv}/bin/${freezecmd} freeze
|
||||
echo "======================================================================"
|
||||
|
||||
MARKER_TEXT="Project: $ZUUL_PROJECT Ref: $ZUUL_REFNAME Build: $ZUUL_UUID Revision: $ZUUL_NEWREV"
|
||||
echo $MARKER_TEXT > doc/build/html/.root-marker
|
||||
|
||||
if [ -z "$ZUUL_REFNAME" ] || [ "$ZUUL_REFNAME" == "master" ] ; then
|
||||
: # Leave the docs where they are.
|
||||
elif echo $ZUUL_REFNAME | grep refs/tags/ >/dev/null ; then
|
||||
# Put tagged releases in proper location. All tagged builds get copied to
|
||||
# BUILD_DIR/tagname. If this is the latest tagged release the copy of files
|
||||
# at BUILD_DIR remains. When Jenkins copies this file the root developer
|
||||
# docs are always the latest release with older tags available under the
|
||||
# root in the tagname dir.
|
||||
TAG=$(echo $ZUUL_REFNAME | sed 's/refs.tags.//')
|
||||
if [ ! -z $TAG ] ; then
|
||||
# This is a hack to ignore the year.release tags in projects since
|
||||
# now all projects use semver based versions instead of date based
|
||||
# versions. The date versions will sort higher even though they
|
||||
# should not so we just special case it here.
|
||||
LATEST=$(git tag | sed -n -e '/^20[0-9]\{2\}\..*$/d' -e '/^[0-9]\+\(\.[0-9]\+\)*$/p' | sort -V | tail -1)
|
||||
# Now publish to / and /$TAG if this is the latest version for projects
|
||||
# and we are only publishing from the release pipeline,
|
||||
# or just /$TAG otherwise.
|
||||
if [ "$tags_handling" = "tags-only" -a "$TAG" = "$LATEST" ] ; then
|
||||
# Copy the docs into a subdir if this is a tagged build
|
||||
mkdir doc/build/$TAG
|
||||
cp -R doc/build/html/. doc/build/$TAG
|
||||
mv doc/build/$TAG doc/build/html/$TAG
|
||||
else
|
||||
# Move the docs into a subdir if this is a tagged build
|
||||
mv doc/build/html doc/build/tmp
|
||||
mkdir doc/build/html
|
||||
mv doc/build/tmp doc/build/html/$TAG
|
||||
fi
|
||||
fi
|
||||
elif echo $ZUUL_REFNAME | grep stable/ >/dev/null ; then
|
||||
# Put stable release changes in dir named after stable release under the
|
||||
# build dir. When Jenkins copies these files they will be accessible under
|
||||
# the developer docs root using the stable release's name.
|
||||
BRANCH=$(echo $ZUUL_REFNAME | sed 's/stable.//')
|
||||
if [ ! -z $BRANCH ] ; then
|
||||
# Move the docs into a subdir if this is a stable branch build
|
||||
mv doc/build/html doc/build/tmp
|
||||
mkdir doc/build/html
|
||||
mv doc/build/tmp doc/build/html/$BRANCH
|
||||
fi
|
||||
else
|
||||
# Put other branch changes in dir named after branch under the
|
||||
# build dir. When Jenkins copies these files they will be
|
||||
# accessible under the developer docs root using the branch name.
|
||||
# EG: feature/foo or milestone-proposed
|
||||
BRANCH=$ZUUL_REFNAME
|
||||
TOP=`dirname $BRANCH`
|
||||
mv doc/build/html doc/build/tmp
|
||||
mkdir -p doc/build/html/$TOP
|
||||
mv doc/build/tmp doc/build/html/$BRANCH
|
||||
fi
|
||||
|
||||
exit $result
|
||||
@ -1,43 +0,0 @@
|
||||
#!/bin/bash -x
|
||||
|
||||
# Copyright 2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
venv=${1:-pep8}
|
||||
|
||||
export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
|
||||
|
||||
virtualenv sdist_check && sdist_check/bin/pip install pbr && sdist_check/bin/python setup.py sdist
|
||||
sdistrc=$?
|
||||
rm -rf sdist_check
|
||||
|
||||
tox -v -e$venv
|
||||
rc=$?
|
||||
|
||||
[ -e .tox/$venv/bin/pbr ] && freezecmd=pbr || freezecmd=pip
|
||||
|
||||
echo "Begin $freezecmd freeze output from test virtualenv:"
|
||||
echo "======================================================================"
|
||||
.tox/${venv}/bin/${freezecmd} freeze
|
||||
echo "======================================================================"
|
||||
|
||||
if [ ! $sdistrc ] ; then
|
||||
echo "******************************************************************"
|
||||
echo "Project cannot create sdist tarball!!!"
|
||||
echo "To reproduce this locally, run: 'python setup.py sdist'"
|
||||
echo "******************************************************************"
|
||||
exit $sdistrc
|
||||
fi
|
||||
|
||||
exit $rc
|
||||
@ -1,23 +0,0 @@
|
||||
#!/bin/bash -xe
|
||||
|
||||
# Copyright 2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
venv=${1:-pylint}
|
||||
export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
|
||||
|
||||
set -o pipefail
|
||||
tox -v -e$venv | tee pylint.txt
|
||||
set +o pipefail
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
#!/bin/bash -xe
|
||||
|
||||
# If a bundle file is present, call tox with the jenkins version of
|
||||
# the test environment so it is used. Otherwise, use the normal
|
||||
# (non-bundle) test environment. Also, run pbr freeze on the
|
||||
# resulting environment at the end so that we have a record of exactly
|
||||
# what packages we ended up testing.
|
||||
#
|
||||
|
||||
venv=venv
|
||||
|
||||
VDISPLAY=99
|
||||
DIMENSIONS='1280x1024x24'
|
||||
/usr/bin/Xvfb :${VDISPLAY} -screen 0 ${DIMENSIONS} 2>&1 > /dev/null &
|
||||
|
||||
set +e
|
||||
DISPLAY=:${VDISPLAY} NOSE_WITH_XUNIT=1 tox -e$venv -- \
|
||||
/bin/bash run_tests.sh -N --only-selenium
|
||||
result=$?
|
||||
|
||||
pkill Xvfb 2>&1 > /dev/null
|
||||
set -e
|
||||
|
||||
[ -e .tox/$venv/bin/pbr ] && freezecmd=pbr || freezecmd=pip
|
||||
|
||||
echo "Begin $freezecmd freeze output from test virtualenv:"
|
||||
echo "======================================================================"
|
||||
.tox/${venv}/bin/${freezecmd} freeze
|
||||
echo "======================================================================"
|
||||
|
||||
exit $result
|
||||
@ -1,33 +0,0 @@
|
||||
#!/bin/bash -xe
|
||||
|
||||
# Copyright 2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
venv=${1:-venv}
|
||||
|
||||
export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
|
||||
|
||||
rm -f dist/*.tar.gz
|
||||
tox -e$venv python setup.py sdist
|
||||
|
||||
FILES=dist/*.tar.gz
|
||||
for f in $FILES; do
|
||||
echo "SHA1sum for $f:"
|
||||
sha1sum $f | awk '{print $1}' > $f.sha1
|
||||
cat $f.sha1
|
||||
|
||||
echo "MD5sum for $f:"
|
||||
md5sum $f | awk '{print $1}' > $f.md5
|
||||
cat $f.md5
|
||||
done
|
||||
@ -1,32 +0,0 @@
|
||||
#!/bin/bash -xe
|
||||
|
||||
# Copyright 2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
venv=${1:-venv}
|
||||
|
||||
export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
|
||||
|
||||
rm -f dist/*.whl
|
||||
tox -e$venv pip install wheel
|
||||
tox -e$venv python setup.py bdist_wheel
|
||||
|
||||
FILES=dist/*.whl
|
||||
for f in $FILES; do
|
||||
echo -n "SHA1sum for $f: "
|
||||
sha1sum $f | awk '{print $1}' | tee $f.sha1
|
||||
|
||||
echo -n "MD5sum for $f: "
|
||||
md5sum $f | awk '{print $1}' | tee $f.md5
|
||||
done
|
||||
@ -1,46 +0,0 @@
|
||||
#!/bin/bash -xe
|
||||
#
|
||||
# Copyright 2012 Hewlett-Packard Development Company, L.P.
|
||||
# Copyright 2013, 2016 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
# Retrieve python tarballs/wheels and make detached OpenPGP signatures.
|
||||
|
||||
PROJECT=$1
|
||||
TARBALL_SITE=$2
|
||||
TAG=$(echo $ZUUL_REF | sed 's/^refs.tags.//')
|
||||
|
||||
# Look in the setup.cfg to determine if a package name is specified, but
|
||||
# fall back on the project name if necessary. Also look in the setup.cfg
|
||||
# to see if this is a universal wheel or not
|
||||
DISTNAME=$(/usr/local/jenkins/slave_scripts/pypi-extract-name.py --wheel \
|
||||
|| echo $PROJECT)
|
||||
TARBALL=$(/usr/local/jenkins/slave_scripts/pypi-extract-name.py \
|
||||
--tarball || echo $PROJECT)-${TAG}.tar.gz
|
||||
WHEEL=$(/usr/local/jenkins/slave_scripts/pypi-extract-name.py \
|
||||
--wheel || echo $PROJECT)-${TAG}-$( \
|
||||
/usr/local/jenkins/slave_scripts/pypi-extract-universal.py || \
|
||||
true)-none-any.whl
|
||||
|
||||
rm -rf *.asc *.tar.gz *.whl
|
||||
|
||||
curl --fail -o $TARBALL https://${TARBALL_SITE}/${PROJECT}/${TARBALL}
|
||||
file -b $TARBALL | grep gzip # Make sure we actually got a tarball
|
||||
gpg --armor --detach-sign $TARBALL
|
||||
|
||||
# Wheels are not mandatory, so only sign if we have one
|
||||
if curl --fail -o $WHEEL https://${TARBALL_SITE}/${PROJECT}/${WHEEL}; then
|
||||
file -b $WHEEL | grep -i zip # Make sure we actually got a wheel
|
||||
gpg --armor --detach-sign $WHEEL
|
||||
fi
|
||||
@ -1,200 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# tardiff.py -- compare the tar package with git archive. Error out if
|
||||
# it's different. The files to exclude are stored in a file, one per line,
|
||||
# and it's passed as argument to this script.
|
||||
#
|
||||
# You should run this script from the project directory. For example, if
|
||||
# you are verifying the package for glance project, you should run this
|
||||
# script from that directory.
|
||||
|
||||
import getopt
|
||||
import sys
|
||||
import os
|
||||
import commands
|
||||
|
||||
|
||||
class OpenStackTarDiff:
|
||||
""" main class to verify tar generated in each openstack projects """
|
||||
|
||||
def __init__(self):
|
||||
self.init_vars()
|
||||
self.validate_args()
|
||||
self.check_env()
|
||||
|
||||
def check_env(self):
|
||||
""" exit if dist/ directory already exists """
|
||||
if not self.package and os.path.exists(self.dist_dir):
|
||||
self.error(
|
||||
"dist directory '%s' exist. Please remove it before "
|
||||
"running this script" % self.dist_dir)
|
||||
|
||||
def validate_args(self):
|
||||
try:
|
||||
opts = getopt.getopt(sys.argv[1:], 'hvp:e:',
|
||||
['help', 'verbose', 'package=',
|
||||
'exclude='])[0]
|
||||
except getopt.GetoptError:
|
||||
self.usage('invalid option selected')
|
||||
|
||||
for opt, value in opts:
|
||||
if (opt in ('-h', '--help')):
|
||||
self.usage()
|
||||
elif (opt in ('-e', '--exclude')):
|
||||
self.e_file = value
|
||||
elif (opt in ('-p', '--package')):
|
||||
self.package = value
|
||||
elif (opt in ('-v', '--verbose')):
|
||||
self.verbose = True
|
||||
else:
|
||||
self.usage('unknown option : ' + opt)
|
||||
if not self.e_file:
|
||||
self.usage('specify file name containing list of files to '
|
||||
'exclude in tar diff')
|
||||
if not os.path.exists(self.e_file):
|
||||
self.usage("file '%s' does not exist" % self.e_file)
|
||||
if self.package and not os.path.exists(self.package):
|
||||
self.usage("package '%s' specified, but does not "
|
||||
"exist" % self.package)
|
||||
|
||||
def init_vars(self):
|
||||
self.dist_dir = 'dist/'
|
||||
self.verbose = False
|
||||
|
||||
self.e_file = None
|
||||
self.project_name = None
|
||||
self.prefix = None
|
||||
self.package = None
|
||||
self.sdist_files = []
|
||||
self.exclude_files = []
|
||||
self.git_files = []
|
||||
self.missing_files = []
|
||||
|
||||
def verify(self):
|
||||
self.get_exclude_files()
|
||||
self.get_project_name()
|
||||
self.get_sdist_files()
|
||||
self.prefix = self.sdist_files[0]
|
||||
self.get_git_files()
|
||||
|
||||
for file in self.git_files:
|
||||
if os.path.basename(file) in self.exclude_files:
|
||||
self.debug("excluding file '%s'" % file)
|
||||
continue
|
||||
|
||||
if file not in self.sdist_files:
|
||||
self.missing_files.append(file)
|
||||
else:
|
||||
# self.debug("file %s matches" % file)
|
||||
pass
|
||||
if len(self.missing_files) > 0:
|
||||
self.error("files missing in package: %s" % self.missing_files)
|
||||
print("SUCCESS: Generated package '%s' is valid" % self.package)
|
||||
|
||||
def get_project_name(self):
|
||||
""" get git project name """
|
||||
self.project_name = os.path.basename(os.path.abspath(os.curdir))
|
||||
|
||||
def get_exclude_files(self):
|
||||
""" read the file and get file list """
|
||||
fh = open(self.e_file, 'r')
|
||||
content = fh.readlines()
|
||||
fh.close()
|
||||
self.debug("files to exclude: %s" % content)
|
||||
|
||||
# remove trailing new lines.
|
||||
self.exclude_files = [x.strip() for x in content]
|
||||
|
||||
def get_git_files(self):
|
||||
""" read file list from git archive """
|
||||
git_tar = os.path.join(os.getcwd(), '%s.tar' % self.project_name)
|
||||
try:
|
||||
a_cmd = ("git archive -o %s HEAD --prefix=%s" %
|
||||
(git_tar, self.prefix))
|
||||
self.debug("executing command '%s'" % a_cmd)
|
||||
(status, out) = commands.getstatusoutput(a_cmd)
|
||||
if status != 0:
|
||||
self.debug("command '%s' returned status '%s'" %
|
||||
(a_cmd, status))
|
||||
if os.path.exists(git_tar):
|
||||
os.unlink(git_tar)
|
||||
self.error('git archive failed: %s' % out)
|
||||
except Exception as err:
|
||||
if os.path.exists(git_tar):
|
||||
os.unlink(git_tar)
|
||||
self.error('git archive failed: %s' % err)
|
||||
|
||||
try:
|
||||
tar_cmd = "tar tf %s" % git_tar
|
||||
self.debug("executing command '%s'" % tar_cmd)
|
||||
(status, out) = commands.getstatusoutput(tar_cmd)
|
||||
if status != 0:
|
||||
self.error('invalid tar file: %s' % git_tar)
|
||||
self.git_files = out.split('\n')
|
||||
self.debug("Removing git archive ... %s ..." % git_tar)
|
||||
os.remove(git_tar)
|
||||
except Exception as err:
|
||||
self.error('unable to read tar: %s' % err)
|
||||
|
||||
def get_sdist_files(self):
|
||||
""" create package for project and get file list in it"""
|
||||
if not self.package:
|
||||
try:
|
||||
sdist_cmd = "python setup.py sdist"
|
||||
self.debug("executing command '%s'" % sdist_cmd)
|
||||
(status, out) = commands.getstatusoutput(sdist_cmd)
|
||||
if status != 0:
|
||||
self.error("command '%s' failed" % sdist_cmd)
|
||||
except Exception as err:
|
||||
self.error("command '%s' failed" % (sdist_cmd, err))
|
||||
|
||||
self.package = os.listdir(self.dist_dir)[0]
|
||||
self.package = os.path.join(self.dist_dir, self.package)
|
||||
tar_cmd = "tar tzf %s" % self.package
|
||||
try:
|
||||
self.debug("executing command '%s'" % tar_cmd)
|
||||
(status, out) = commands.getstatusoutput(tar_cmd)
|
||||
if status != 0:
|
||||
self.error("command '%s' failed" % tar_cmd)
|
||||
# self.debug(out)
|
||||
self.sdist_files = out.split('\n')
|
||||
except Exception as err:
|
||||
self.error("command '%s' failed: %s" % (tar_cmd, err))
|
||||
|
||||
def debug(self, msg):
|
||||
if self.verbose:
|
||||
sys.stdout.write('DEBUG: %s\n' % msg)
|
||||
|
||||
def error(self, msg):
|
||||
sys.stderr.write('ERROR: %s\n' % msg)
|
||||
sys.exit(1)
|
||||
|
||||
def usage(self, msg=None):
|
||||
if msg:
|
||||
stream = sys.stderr
|
||||
else:
|
||||
stream = sys.stdout
|
||||
stream.write("usage: %s [--help|h] [-v] "
|
||||
"[-p|--package=sdist_package.tar.gz] "
|
||||
"-e|--exclude=filename\n" % os.path.basename(sys.argv[0]))
|
||||
if msg:
|
||||
stream.write("\nERROR: " + msg + "\n")
|
||||
exitCode = 1
|
||||
else:
|
||||
exitCode = 0
|
||||
sys.exit(exitCode)
|
||||
|
||||
if __name__ == '__main__':
|
||||
tardiff = OpenStackTarDiff()
|
||||
tardiff.verify()
|
||||
Loading…
Reference in New Issue
Block a user