project-config/jenkins/scripts/project-requirements-change.py
Jeremy Stanley 0fa17bce9c Only error for duplicate requirements when strict
If a project adds a requirements check job while its requirements
files are nonconforming, they should be able to fix it by submitting
a change which fixes their requirements. If validity checks get
performed while reading requirements lists in the
gate{name}-requirements job, they need to only trigger failure while
parsing the submitted requirements and not on the existing state of
requirements before applying a proposed change.

Pass the "strict" flag through and wrap the duplicate entries check
in a conditional for it, so that we don't block fixes for projects
that start out with a duplicate requirement.

Change-Id: I16cbea930fd104717e73bd4e012ec77fc4f98494
2016-08-08 15:42:29 +00:00

262 lines
10 KiB
Python
Executable File

#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
# Copyright (c) 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import collections
import contextlib
import os
import shlex
import shutil
import subprocess
import sys
import tempfile
requirement = None
project = None
def run_command(cmd):
print(cmd)
cmd_list = shlex.split(str(cmd))
p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(out, err) = p.communicate()
if p.returncode != 0:
raise SystemError(err)
return (out.strip(), err.strip())
class RequirementsList(object):
def __init__(self, name, project):
self.name = name
self.reqs_by_file = {}
self.project = project
self.failed = False
@property
def reqs(self):
return {k: v for d in self.reqs_by_file.values()
for k, v in d.items()}
def extract_reqs(self, content, strict):
reqs = collections.defaultdict(set)
parsed = requirement.parse(content)
for name, entries in parsed.items():
if not name:
# Comments and other unprocessed lines
continue
list_reqs = [r for (r, line) in entries]
# Strip the comments out before checking if there are duplicates
list_reqs_stripped = [r._replace(comment='') for r in list_reqs]
if strict and len(list_reqs_stripped) != len(set(
list_reqs_stripped)):
print("Requirements file has duplicate entries "
"for package %s : %r." % (name, list_reqs))
self.failed = True
reqs[name].update(list_reqs)
return reqs
def process(self, strict=True):
"""Convert the project into ready to use data.
- an iterable of requirement sets to check
- each set has the following rules:
- each has a list of Requirements objects
- duplicates are not permitted within that list
"""
print("Checking %(name)s" % {'name': self.name})
# First, parse.
for fname, content in self.project.get('requirements', {}).items():
print("Processing %(fname)s" % {'fname': fname})
if strict and not content.endswith('\n'):
print("Requirements file %s does not "
"end with a newline." % fname)
self.reqs_by_file[fname] = self.extract_reqs(content, strict)
for name, content in project.extras(self.project).items():
print("Processing .[%(extra)s]" % {'extra': name})
self.reqs_by_file[name] = self.extract_reqs(content, strict)
def grab_args():
"""Grab and return arguments"""
parser = argparse.ArgumentParser(
description="Check if project requirements have changed"
)
parser.add_argument('--local', action='store_true',
help='check local changes (not yet in git)')
parser.add_argument('branch', nargs='?', default='master',
help='target branch for diffs')
parser.add_argument('--zc', help='what zuul cloner to call')
parser.add_argument('--reqs', help='use a specified requirements tree')
return parser.parse_args()
@contextlib.contextmanager
def tempdir():
try:
reqroot = tempfile.mkdtemp()
yield reqroot
finally:
shutil.rmtree(reqroot)
def install_and_load_requirements(reqroot, reqdir):
sha = run_command("git --git-dir %s/.git rev-parse HEAD" % reqdir)[0]
print "requirements git sha: %s" % sha
req_venv = os.path.join(reqroot, 'venv')
req_pip = os.path.join(req_venv, 'bin/pip')
req_lib = os.path.join(req_venv, 'lib/python2.7/site-packages')
out, err = run_command("virtualenv " + req_venv)
out, err = run_command(req_pip + " install " + reqdir)
sys.path.append(req_lib)
global project
global requirement
from openstack_requirements import project # noqa
from openstack_requirements import requirement # noqa
def _is_requirement_in_global_reqs(req, global_reqs):
# Compare all fields except the extras field as the global
# requirements should not have any lines with the extras syntax
# example: oslo.db[xyz]<1.2.3
for req2 in global_reqs:
if (req.package == req2.package and
req.location == req2.location and
req.specifiers == req2.specifiers and
req.markers == req2.markers and
req.comment == req2.comment):
return True
return False
def main():
args = grab_args()
branch = args.branch
failed = False
# build a list of requirements from the global list in the
# openstack/requirements project so we can match them to the changes
with tempdir() as reqroot:
# Only clone requirements repo if no local repo is specified
# on the command line.
if args.reqs is None:
reqdir = os.path.join(reqroot, "openstack/requirements")
if args.zc is not None:
zc = args.zc
else:
zc = '/usr/zuul-env/bin/zuul-cloner'
out, err = run_command("%(zc)s "
"--cache-dir /opt/git "
"--workspace %(root)s "
"git://git.openstack.org "
"openstack/requirements"
% dict(zc=zc, root=reqroot))
print out
print err
else:
reqdir = args.reqs
install_and_load_requirements(reqroot, reqdir)
global_reqs = requirement.parse(
open(reqdir + '/global-requirements.txt', 'rt').read())
for k, entries in global_reqs.items():
# Discard the lines: we don't need them.
global_reqs[k] = set(r for (r, line) in entries)
cwd = os.getcwd()
# build a list of requirements in the proposed change,
# and check them for style violations while doing so
head = run_command("git rev-parse HEAD")[0]
head_proj = project.read(cwd)
head_reqs = RequirementsList('HEAD', head_proj)
# Don't apply strict parsing rules to stable branches.
# Reasoning is:
# - devstack etc protect us from functional issues
# - we're backporting to stable, so guarding against
# aesthetics and DRY concerns is not our business anymore
# - if in future we have other not-functional linty style
# things to add, we don't want them to affect stable
# either.
head_strict = not branch.startswith('stable/')
head_reqs.process(strict=head_strict)
if not args.local:
# build a list of requirements already in the target branch,
# so that we can create a diff and identify what's being changed
run_command("git remote update")
run_command("git checkout remotes/origin/%s" % branch)
branch_proj = project.read(cwd)
# switch back to the proposed change now
run_command("git checkout %s" % head)
else:
branch_proj = {'root': cwd}
branch_reqs = RequirementsList(branch, branch_proj)
# Don't error on the target branch being broken.
branch_reqs.process(strict=False)
# iterate through the changing entries and see if they match the global
# equivalents we want enforced
for fname, freqs in head_reqs.reqs_by_file.items():
print("Validating %(fname)s" % {'fname': fname})
for name, reqs in freqs.items():
counts = {}
if (name in branch_reqs.reqs and
reqs == branch_reqs.reqs[name]):
# Unchanged [or a change that preserves a current value]
continue
if name not in global_reqs:
failed = True
print("Requirement %s not in openstack/requirements" %
str(reqs))
continue
if reqs == global_reqs[name]:
continue
for req in reqs:
if req.extras:
for extra in req.extras:
counts[extra] = counts.get(extra, 0) + 1
else:
counts[''] = counts.get('', 0) + 1
if not _is_requirement_in_global_reqs(
req, global_reqs[name]):
failed = True
print("Requirement for package %s : %s does "
"not match openstack/requirements value : %s" % (
name, str(req), str(global_reqs[name])))
for extra, count in counts.items():
if count != len(global_reqs[name]):
failed = True
print("Package %s%s requirement does not match "
"number of lines (%d) in "
"openstack/requirements" % (
name,
('[%s]' % extra) if extra else '',
len(global_reqs[name])))
# report the results
if failed or head_reqs.failed or branch_reqs.failed:
sys.exit(1)
print("Updated requirements match openstack/requirements.")
if __name__ == '__main__':
main()