c671d3b0c1
Depends-On: I90362ea9968f423f0e6aeb638adb0e38bdf097d0 Change-Id: I14dab0936e3bae196f29e867f4987f6b38c728aa Signed-off-by: Doug Hellmann <doug@doughellmann.com>
307 lines
12 KiB
Python
307 lines
12 KiB
Python
# Copyright 2012 OpenStack Foundation
|
|
# Copyright 2013 Hewlett-Packard Development Company, L.P.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""
|
|
A simple script to update the requirements files from a global set of
|
|
allowable requirements.
|
|
|
|
The script can be called like this:
|
|
|
|
$> python update.py ../myproj
|
|
|
|
Any requirements listed in the target files will have their versions
|
|
updated to match the global requirements. Requirements not in the global
|
|
files will be dropped.
|
|
"""
|
|
|
|
import optparse
|
|
import os
|
|
import os.path
|
|
import sys
|
|
|
|
import six
|
|
|
|
from openstack_requirements import project
|
|
from openstack_requirements import requirement
|
|
|
|
_setup_py_text = """# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
|
|
import setuptools
|
|
|
|
# In python < 2.7.4, a lazy loading of package `pbr` will break
|
|
# setuptools if some other modules registered functions in `atexit`.
|
|
# solution from: http://bugs.python.org/issue15881#msg170215
|
|
try:
|
|
import multiprocessing # noqa
|
|
except ImportError:
|
|
pass
|
|
|
|
setuptools.setup(
|
|
setup_requires=['pbr>=2.0.0'],
|
|
pbr=True)
|
|
"""
|
|
|
|
|
|
# Pure --
|
|
|
|
class Change(object):
|
|
def __init__(self, name, old, new):
|
|
self.name = name
|
|
self.old = old.strip()
|
|
self.new = new.strip()
|
|
|
|
def __repr__(self):
|
|
return "%-30.30s -> %s" % (self.old, self.new)
|
|
|
|
|
|
def _check_setup_py(proj):
|
|
actions = []
|
|
# If it doesn't have a setup.py, then we don't want to update it
|
|
if 'setup.py' not in proj:
|
|
return actions
|
|
# If it doesn't use pbr, we don't want to update it.
|
|
elif 'pbr' not in proj['setup.py']:
|
|
return actions
|
|
# We don't update pbr's setup.py because it can't use itself.
|
|
if 'setup.cfg' in proj and 'name = pbr' in proj['setup.cfg']:
|
|
return actions
|
|
actions.append(project.Verbose("Syncing setup.py"))
|
|
actions.append(project.File('setup.py', _setup_py_text))
|
|
return actions
|
|
|
|
|
|
def _sync_requirements_file(
|
|
source_reqs, dest_sequence, dest_label, softupdate, hacking,
|
|
non_std_reqs, blacklist={}):
|
|
actions = []
|
|
dest_reqs = requirement.to_dict(dest_sequence)
|
|
changes = []
|
|
output_requirements = []
|
|
processed_packages = set()
|
|
|
|
for req, req_line in dest_sequence:
|
|
# Skip the instructions header
|
|
if req_line in requirement._REQS_HEADER:
|
|
continue
|
|
elif req is None:
|
|
# Unparsable lines.
|
|
output_requirements.append(
|
|
requirement.Requirement('', '', '', '', req_line.rstrip()))
|
|
continue
|
|
elif not req.package:
|
|
# Comment-only lines
|
|
output_requirements.append(req)
|
|
continue
|
|
elif req.package.lower() in processed_packages:
|
|
continue
|
|
|
|
processed_packages.add(req.package.lower())
|
|
# Special cases:
|
|
# projects need to align hacking version on their own time
|
|
if req.package == "hacking" and not hacking:
|
|
output_requirements.append(req)
|
|
continue
|
|
# the overall blacklist is similarly synced by projects as
|
|
# needed
|
|
if req.package in blacklist:
|
|
output_requirements.append(req)
|
|
continue
|
|
|
|
reference = source_reqs.get(req.package.lower())
|
|
if reference:
|
|
actual = dest_reqs.get(req.package.lower())
|
|
for req, ref in six.moves.zip_longest(actual, reference):
|
|
if not req:
|
|
# More in globals
|
|
changes.append(Change(ref[0].package, '', ref[1]))
|
|
elif not ref:
|
|
# less in globals
|
|
changes.append(Change(req[0].package, req[1], ''))
|
|
elif req[0] != ref[0]:
|
|
# NOTE(jamielennox): extras are allowed to be specified in
|
|
# a project's requirements and the version be updated and
|
|
# extras maintained. Create a new ref object the same as
|
|
# the original but with the req's extras.
|
|
|
|
merged_ref = requirement.Requirement(ref[0].package,
|
|
ref[0].location,
|
|
ref[0].specifiers,
|
|
ref[0].markers,
|
|
ref[0].comment,
|
|
req[0].extras)
|
|
|
|
ref = (merged_ref, merged_ref.to_line())
|
|
|
|
if req[0] != ref[0]:
|
|
# A change on this entry
|
|
changes.append(Change(req[0].package, req[1], ref[1]))
|
|
|
|
if ref:
|
|
output_requirements.append(ref[0])
|
|
elif softupdate:
|
|
# under softupdate we pass through anything unknown packages,
|
|
# this is intended for ecosystem projects that want to stay in
|
|
# sync with existing requirements, but also add their own above
|
|
# and beyond.
|
|
output_requirements.append(req)
|
|
else:
|
|
# What do we do if we find something unexpected?
|
|
#
|
|
# In the default cause we should die horribly, because
|
|
# the point of global requirements was a single lever
|
|
# to control all the pip installs in the gate.
|
|
#
|
|
# However, we do have other projects using
|
|
# devstack jobs that might have legitimate reasons to
|
|
# override. For those we support NON_STANDARD_REQS=1
|
|
# environment variable to turn this into a warning only.
|
|
# However this drops the unknown requirement.
|
|
actions.append(project.Error(
|
|
"'%s' is not in global-requirements.txt or blacklist.txt"
|
|
% req.package))
|
|
# always print out what we did if we did a thing
|
|
if changes:
|
|
actions.append(project.StdOut(
|
|
"Version change for: %s\n"
|
|
% ", ".join([x.name for x in changes])))
|
|
actions.append(project.StdOut("Updated %s:\n" % dest_label))
|
|
for change in changes:
|
|
actions.append(project.StdOut(" %s\n" % change))
|
|
return actions, requirement.Requirements(output_requirements)
|
|
|
|
|
|
def _copy_requires(
|
|
suffix, softupdate, hacking, proj, global_reqs, non_std_reqs,
|
|
blacklist={}):
|
|
"""Copy requirements files."""
|
|
actions = []
|
|
for source, content in sorted(proj['requirements'].items()):
|
|
dest_path = os.path.join(proj['root'], source)
|
|
# this is specifically for global-requirements gate jobs so we don't
|
|
# modify the git tree
|
|
if suffix:
|
|
dest_path = "%s.%s" % (dest_path, suffix)
|
|
dest_name = "%s.%s" % (source, suffix)
|
|
else:
|
|
dest_name = source
|
|
dest_sequence = list(requirement.to_reqs(content))
|
|
actions.append(project.Verbose("Syncing %s" % dest_path))
|
|
_actions, reqs = _sync_requirements_file(
|
|
global_reqs, dest_sequence, dest_path, softupdate, hacking,
|
|
non_std_reqs, blacklist)
|
|
actions.extend(_actions)
|
|
actions.append(project.File(dest_name, requirement.to_content(reqs)))
|
|
extras = project.extras(proj)
|
|
output_extras = {}
|
|
for extra, content in sorted(extras.items()):
|
|
dest_name = 'extra-%s' % extra
|
|
dest_path = "%s[%s]" % (proj['root'], extra)
|
|
dest_sequence = list(requirement.to_reqs(content))
|
|
actions.append(project.Verbose("Syncing extra [%s]" % extra))
|
|
_actions, reqs = _sync_requirements_file(
|
|
global_reqs, dest_sequence, dest_path, softupdate, hacking,
|
|
non_std_reqs, blacklist)
|
|
actions.extend(_actions)
|
|
output_extras[extra] = reqs
|
|
dest_path = 'setup.cfg'
|
|
if suffix:
|
|
dest_path = "%s.%s" % (dest_path, suffix)
|
|
actions.append(project.File(
|
|
dest_path, project.merge_setup_cfg(proj['setup.cfg'], output_extras)))
|
|
return actions
|
|
|
|
|
|
def _process_project(
|
|
project, global_reqs, suffix, softupdate, hacking, non_std_reqs,
|
|
blacklist={}):
|
|
"""Project a project.
|
|
|
|
:return: The actions to take as a result.
|
|
"""
|
|
actions = _copy_requires(
|
|
suffix, softupdate, hacking, project, global_reqs, non_std_reqs,
|
|
blacklist)
|
|
actions.extend(_check_setup_py(project))
|
|
return actions
|
|
|
|
|
|
# IO --
|
|
|
|
def main(argv=None, stdout=None, _worker=None):
|
|
parser = optparse.OptionParser()
|
|
parser.add_option("-o", "--output-suffix", dest="suffix", default="",
|
|
help="output suffix for updated files (i.e. .global)")
|
|
parser.add_option("-s", "--soft-update", dest="softupdate",
|
|
action="store_true",
|
|
help="Pass through extra requirements without warning.")
|
|
parser.add_option("-H", "--hacking", dest="hacking",
|
|
action="store_true",
|
|
help="Include the hacking project.")
|
|
parser.add_option("-v", "--verbose", dest="verbose",
|
|
action="store_true",
|
|
help="Add further verbosity to output")
|
|
parser.add_option("--source", dest="source", default=".",
|
|
help="Dir where global-requirements.txt is located.")
|
|
options, args = parser.parse_args(argv)
|
|
if len(args) != 1:
|
|
print("Must specify directory to update")
|
|
raise Exception("Must specify one and only one directory to update.")
|
|
if not os.path.isdir(args[0]):
|
|
print("%s is not a directory." % (args[0]))
|
|
raise Exception("%s is not a directory." % (args[0]))
|
|
if stdout is None:
|
|
stdout = sys.stdout
|
|
if _worker is None:
|
|
_worker = _do_main
|
|
non_std_reqs = os.getenv('NON_STANDARD_REQS', '0') == '1'
|
|
_worker(
|
|
args[0], options.source, options.suffix, options.softupdate,
|
|
options.hacking, stdout, options.verbose, non_std_reqs)
|
|
|
|
|
|
def _do_main(
|
|
root, source, suffix, softupdate, hacking, stdout, verbose,
|
|
non_std_reqs):
|
|
"""No options or environment variable access from here on in."""
|
|
proj = project.read(root)
|
|
global_req_content = open(
|
|
os.path.join(source, 'global-requirements.txt'), 'rt').read()
|
|
global_reqs = requirement.parse(global_req_content)
|
|
blacklist_content = open(
|
|
os.path.join(source, 'blacklist.txt'), 'rt').read()
|
|
blacklist = requirement.parse(blacklist_content)
|
|
actions = _process_project(
|
|
proj, global_reqs, suffix, softupdate, hacking, non_std_reqs,
|
|
blacklist)
|
|
project.write(proj, actions, stdout=stdout, verbose=verbose)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|