Global requirements for OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
requirements/openstack_requirements/cmds/update.py

298 lines
11 KiB

# Copyright 2012 OpenStack Foundation
# Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
A simple script to update the requirements files from a global set of
allowable requirements.
The script can be called like this:
$> python update.py ../myproj
Any requirements listed in the target files will have their versions
updated to match the global requirements. Requirements not in the global
files will be dropped.
"""
import optparse
import os
import os.path
import sys
import six
from openstack_requirements import project
from openstack_requirements import requirement
_setup_py_text = """# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
import setuptools
setuptools.setup(
setup_requires=['pbr>=2.0.0'],
pbr=True)
"""
# Pure --
class Change(object):
def __init__(self, name, old, new):
self.name = name
self.old = old.strip()
self.new = new.strip()
def __repr__(self):
return "%-30.30s -> %s" % (self.old, self.new)
def _check_setup_py(proj):
actions = []
# If it doesn't have a setup.py, then we don't want to update it
if 'setup.py' not in proj:
return actions
# If it doesn't use pbr, we don't want to update it.
elif 'pbr' not in proj['setup.py']:
return actions
# We don't update pbr's setup.py because it can't use itself.
if 'setup.cfg' in proj and 'name = pbr' in proj['setup.cfg']:
return actions
actions.append(project.Verbose("Syncing setup.py"))
actions.append(project.File('setup.py', _setup_py_text))
return actions
def _sync_requirements_file(
source_reqs, dest_sequence, dest_label, softupdate, hacking,
non_std_reqs, blacklist={}):
actions = []
dest_reqs = requirement.to_dict(dest_sequence)
changes = []
output_requirements = []
processed_packages = set()
for req, req_line in dest_sequence:
# Skip the instructions header
if req_line in requirement._REQS_HEADER:
continue
elif req is None:
# Unparsable lines.
output_requirements.append(
requirement.Requirement('', '', '', '', req_line.rstrip()))
continue
elif not req.package:
# Comment-only lines
output_requirements.append(req)
continue
elif req.package.lower() in processed_packages:
continue
processed_packages.add(req.package.lower())
# Special cases:
# projects need to align hacking version on their own time
if req.package == "hacking" and not hacking:
output_requirements.append(req)
continue
# the overall blacklist is similarly synced by projects as
# needed
if req.package in blacklist:
output_requirements.append(req)
continue
reference = source_reqs.get(req.package.lower())
if reference:
actual = dest_reqs.get(req.package.lower())
for req, ref in six.moves.zip_longest(actual, reference):
if not req:
# More in globals
changes.append(Change(ref[0].package, '', ref[1]))
elif not ref:
# less in globals
changes.append(Change(req[0].package, req[1], ''))
elif req[0] != ref[0]:
# NOTE(jamielennox): extras are allowed to be specified in
# a project's requirements and the version be updated and
# extras maintained. Create a new ref object the same as
# the original but with the req's extras.
merged_ref = requirement.Requirement(ref[0].package,
ref[0].location,
ref[0].specifiers,
ref[0].markers,
ref[0].comment,
req[0].extras)
ref = (merged_ref, merged_ref.to_line())
if req[0] != ref[0]:
# A change on this entry
changes.append(Change(req[0].package, req[1], ref[1]))
if ref:
output_requirements.append(ref[0])
elif softupdate:
# under softupdate we pass through anything unknown packages,
# this is intended for ecosystem projects that want to stay in
# sync with existing requirements, but also add their own above
# and beyond.
output_requirements.append(req)
else:
# What do we do if we find something unexpected?
#
# In the default cause we should die horribly, because
# the point of global requirements was a single lever
# to control all the pip installs in the gate.
#
# However, we do have other projects using
# devstack jobs that might have legitimate reasons to
# override. For those we support NON_STANDARD_REQS=1
# environment variable to turn this into a warning only.
# However this drops the unknown requirement.
actions.append(project.Error(
"'%s' is not in global-requirements.txt or blacklist.txt"
% req.package))
# always print out what we did if we did a thing
if changes:
actions.append(project.StdOut(
"Version change for: %s\n"
% ", ".join([x.name for x in changes])))
actions.append(project.StdOut("Updated %s:\n" % dest_label))
for change in changes:
actions.append(project.StdOut(" %s\n" % change))
return actions, requirement.Requirements(output_requirements)
def _copy_requires(
suffix, softupdate, hacking, proj, global_reqs, non_std_reqs,
blacklist={}):
"""Copy requirements files."""
actions = []
for source, content in sorted(proj['requirements'].items()):
dest_path = os.path.join(proj['root'], source)
# this is specifically for global-requirements gate jobs so we don't
# modify the git tree
if suffix:
dest_path = "%s.%s" % (dest_path, suffix)
dest_name = "%s.%s" % (source, suffix)
else:
dest_name = source
dest_sequence = list(requirement.to_reqs(content))
actions.append(project.Verbose("Syncing %s" % dest_path))
_actions, reqs = _sync_requirements_file(
global_reqs, dest_sequence, dest_path, softupdate, hacking,
non_std_reqs, blacklist)
actions.extend(_actions)
actions.append(project.File(dest_name, requirement.to_content(reqs)))
extras = project.extras(proj)
output_extras = {}
for extra, content in sorted(extras.items()):
dest_name = 'extra-%s' % extra
dest_path = "%s[%s]" % (proj['root'], extra)
dest_sequence = list(requirement.to_reqs(content))
actions.append(project.Verbose("Syncing extra [%s]" % extra))
_actions, reqs = _sync_requirements_file(
global_reqs, dest_sequence, dest_path, softupdate, hacking,
non_std_reqs, blacklist)
actions.extend(_actions)
output_extras[extra] = reqs
dest_path = 'setup.cfg'
if suffix:
dest_path = "%s.%s" % (dest_path, suffix)
actions.append(project.File(
dest_path, project.merge_setup_cfg(proj['setup.cfg'], output_extras)))
return actions
def _process_project(
project, global_reqs, suffix, softupdate, hacking, non_std_reqs,
blacklist={}):
"""Project a project.
:return: The actions to take as a result.
"""
actions = _copy_requires(
suffix, softupdate, hacking, project, global_reqs, non_std_reqs,
blacklist)
actions.extend(_check_setup_py(project))
return actions
# IO --
def main(argv=None, stdout=None, _worker=None):
parser = optparse.OptionParser()
parser.add_option("-o", "--output-suffix", dest="suffix", default="",
help="output suffix for updated files (i.e. .global)")
parser.add_option("-s", "--soft-update", dest="softupdate",
action="store_true",
help="Pass through extra requirements without warning.")
parser.add_option("-H", "--hacking", dest="hacking",
action="store_true",
help="Include the hacking project.")
parser.add_option("-v", "--verbose", dest="verbose",
action="store_true",
help="Add further verbosity to output")
parser.add_option("--source", dest="source", default=".",
help="Dir where global-requirements.txt is located.")
options, args = parser.parse_args(argv)
if len(args) != 1:
print("Must specify directory to update")
raise Exception("Must specify one and only one directory to update.")
if not os.path.isdir(args[0]):
print("%s is not a directory." % (args[0]))
raise Exception("%s is not a directory." % (args[0]))
if stdout is None:
stdout = sys.stdout
if _worker is None:
_worker = _do_main
non_std_reqs = os.getenv('NON_STANDARD_REQS', '0') == '1'
_worker(
args[0], options.source, options.suffix, options.softupdate,
options.hacking, stdout, options.verbose, non_std_reqs)
def _do_main(
root, source, suffix, softupdate, hacking, stdout, verbose,
non_std_reqs):
"""No options or environment variable access from here on in."""
proj = project.read(root)
global_req_content = open(
os.path.join(source, 'global-requirements.txt'), 'rt').read()
global_reqs = requirement.parse(global_req_content)
blacklist_content = open(
os.path.join(source, 'blacklist.txt'), 'rt').read()
blacklist = requirement.parse(blacklist_content)
actions = _process_project(
proj, global_reqs, suffix, softupdate, hacking, non_std_reqs,
blacklist)
project.write(proj, actions, stdout=stdout, verbose=verbose)
if __name__ == "__main__":
main()