8806ac2a53
Previously it seems like we lost requirement urls and then downloading said requirment url would fail (since the url is typically provided if the package isn't on pypi yet). This seems to be happening for the latests nova requirement which seems to try to pull in a special oslo config version. Change-Id: I30acea47f07f6d189fd63ab9e90434f1eb4e4e2d
349 lines
12 KiB
Python
Executable File
349 lines
12 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
import argparse
|
|
import distutils.spawn
|
|
import logging
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
import pip.index
|
|
import pip.req
|
|
from pip.vcs import git, mercurial, subversion, bazaar
|
|
import pkg_resources
|
|
|
|
|
|
BAD_REQUIREMENTS = 2
|
|
INCOMPATIBLE_REQUIREMENTS = 3
|
|
logger = logging.getLogger()
|
|
|
|
|
|
def create_parser():
|
|
parser = argparse.ArgumentParser()
|
|
|
|
parser.add_argument(
|
|
"-r", "--requirement",
|
|
dest="requirements",
|
|
nargs="*",
|
|
default=[],
|
|
metavar="<file>",
|
|
help="Install all the packages listed in the given requirements file")
|
|
parser.add_argument(
|
|
"requirement_specs",
|
|
nargs="*",
|
|
default=[],
|
|
metavar="<requirement specifier>",
|
|
help="Install specified package")
|
|
parser.add_argument(
|
|
# A regex to be used to skip requirements
|
|
"--skip-requirements-regex",
|
|
default="",
|
|
help=argparse.SUPPRESS)
|
|
parser.add_argument(
|
|
# The default version control system for editables, e.g. 'svn'
|
|
'--default-vcs',
|
|
dest='default_vcs',
|
|
default='',
|
|
help=argparse.SUPPRESS)
|
|
parser.add_argument(
|
|
"--debug", "-d",
|
|
action="store_true",
|
|
default=False,
|
|
help="Print debug information")
|
|
parser.add_argument(
|
|
"--ignore-installed", "-i",
|
|
action="store_true",
|
|
default=False,
|
|
help="Ignore installed packages")
|
|
parser.add_argument(
|
|
"--ignore-packages",
|
|
nargs="*",
|
|
default=[],
|
|
metavar="<requirement specifier>",
|
|
help="Ignore listed packages")
|
|
parser.add_argument(
|
|
"--frozen", "-f",
|
|
action="store_true",
|
|
default=False,
|
|
help="Make requirements meet installed packages (taken from pip freeze)")
|
|
pip_executable = (distutils.spawn.find_executable("pip") or
|
|
distutils.spawn.find_executable("pip-python"))
|
|
parser.add_argument(
|
|
"--pip",
|
|
metavar="<filename>",
|
|
default=pip_executable,
|
|
help="Full or short name of pip executable (default: %s)" %
|
|
pip_executable)
|
|
return parser
|
|
|
|
|
|
def setup_logging(options):
|
|
level = logging.DEBUG if options.debug else logging.WARNING
|
|
handler = logging.StreamHandler(sys.stderr)
|
|
logger.addHandler(handler)
|
|
logger.setLevel(level)
|
|
|
|
|
|
incompatibles = set()
|
|
joined_requirements = []
|
|
|
|
|
|
def install_requirement_ensure_req_field(req):
|
|
if not hasattr(req, 'req') or not req.req:
|
|
# pip 0.8 or so
|
|
link = pip.index.Link(req.url)
|
|
name = link.egg_fragment
|
|
if not name:
|
|
raise Exception("Cannot find package name from `%s'" % req.url)
|
|
req.req = pkg_resources.Requirement.parse(name)
|
|
return req
|
|
|
|
|
|
def install_requirement_str(req):
|
|
return req.url or str(req.req)
|
|
|
|
|
|
def install_requirement_parse(line, comes_from):
|
|
line = line.strip()
|
|
if line.startswith('-e') or line.startswith('--editable'):
|
|
if line.startswith('-e'):
|
|
line = line[2:].strip()
|
|
else:
|
|
line = line[len('--editable'):].strip().lstrip('=')
|
|
req = pip.req.InstallRequirement.from_editable(
|
|
line, comes_from=comes_from)
|
|
else:
|
|
req = pip.req.InstallRequirement.from_line(line, comes_from)
|
|
return install_requirement_ensure_req_field(req)
|
|
|
|
|
|
def incompatible_requirement(chosen, conflicting):
|
|
if chosen.req.key not in incompatibles:
|
|
incompatibles.add(chosen.req.key)
|
|
print >> sys.stderr, "%s: incompatible requirements" % chosen.req.key
|
|
print >> sys.stderr, "Choosing:"
|
|
print >> sys.stderr, ("\t%s: %s" %
|
|
(chosen.comes_from,
|
|
install_requirement_str(chosen)))
|
|
print >> sys.stderr, "Conflicting:"
|
|
print >> sys.stderr, ("\t%s: %s" %
|
|
(conflicting.comes_from,
|
|
install_requirement_str(conflicting)))
|
|
|
|
|
|
def parse_requirements(options):
|
|
"""Parse package requirements from command line and files.
|
|
|
|
:return: tuple (all, ignored) of InstallRequirement
|
|
"""
|
|
all_requirements = {}
|
|
skip_match = None
|
|
if options.skip_requirements_regex:
|
|
skip_match = re.compile(options.skip_requirements_regex)
|
|
for req_spec in options.requirement_specs:
|
|
try:
|
|
req = install_requirement_parse(req_spec, "command line")
|
|
if skip_match and skip_match.search(req.req.key):
|
|
continue
|
|
all_requirements.setdefault(req.req.key, []).append(req)
|
|
except Exception as ex:
|
|
logger.error("Cannot parse `%s': %s" % (req_spec, ex))
|
|
sys.exit(BAD_REQUIREMENTS)
|
|
for filename in options.requirements:
|
|
try:
|
|
for req in pip.req.parse_requirements(filename):
|
|
req = install_requirement_ensure_req_field(req)
|
|
if skip_match and skip_match.search(req.req.key):
|
|
continue
|
|
all_requirements.setdefault(req.req.key, []).append(req)
|
|
except Exception as ex:
|
|
logger.error("Cannot parse `%s': %s" % (filename, ex))
|
|
sys.exit(BAD_REQUIREMENTS)
|
|
ignored_requirements = []
|
|
for req_spec in options.ignore_packages:
|
|
try:
|
|
req = install_requirement_parse(req_spec, "command line")
|
|
ignored_requirements.append(req)
|
|
except Exception as ex:
|
|
logger.error("Cannot parse `%s': %s" % (req_spec, ex))
|
|
sys.exit(BAD_REQUIREMENTS)
|
|
return all_requirements, ignored_requirements
|
|
|
|
|
|
def installed_packages(options):
|
|
pip_cmdline = [
|
|
options.pip,
|
|
"freeze",
|
|
]
|
|
(package_list, _) = subprocess.Popen(
|
|
pip_cmdline, stdout=subprocess.PIPE).communicate()
|
|
|
|
pkg_list = []
|
|
for line in package_list.splitlines():
|
|
try:
|
|
pkg_list.append(install_requirement_parse(line, "pip freeze").req)
|
|
except Exception:
|
|
pass
|
|
return pkg_list
|
|
|
|
|
|
def join_one_requirement(req_list):
|
|
"""Join requirement list for one package together.
|
|
|
|
Possible returns:
|
|
* ==A - exact version (even when there are conflicts)
|
|
* >=?A,<=?B,(!=C)+ - line segment (no conflicts detected)
|
|
* >=?A,(!=C)+ - more than (also when conflicts detected)
|
|
|
|
:param:req_list list of pip.req.InstallRequirement
|
|
:return: pip.req.InstallRequirement
|
|
"""
|
|
if len(req_list) == 1:
|
|
return req_list[0]
|
|
req_strict = None
|
|
lower_bound_str = None
|
|
lower_bound_version = None
|
|
lower_bound_req = None
|
|
upper_bound_str = None
|
|
upper_bound_version = None
|
|
upper_bound_req = None
|
|
conflicts = []
|
|
for req in req_list:
|
|
for spec in req.req.specs:
|
|
if spec[0] == "==":
|
|
return req
|
|
spec_str = "%s%s" % spec
|
|
if spec[0] == "!=":
|
|
conflicts.append(spec_str)
|
|
continue
|
|
version = pkg_resources.parse_version(spec[1])
|
|
# strict_check is < or >, not <= or >=
|
|
strict_check = len(spec[0]) == 1
|
|
if spec[0][0] == ">":
|
|
if (not lower_bound_version or (version > lower_bound_version) or
|
|
(strict_check and version == lower_bound_version)):
|
|
lower_bound_version = version
|
|
lower_bound_str = spec_str
|
|
lower_bound_req = req
|
|
else:
|
|
if (not upper_bound_version or (version < upper_bound_version) or
|
|
(strict_check and version == upper_bound_version)):
|
|
upper_bound_version = version
|
|
upper_bound_str = spec_str
|
|
upper_bound_req = req
|
|
if lower_bound_version and upper_bound_version:
|
|
bad_bounds = False
|
|
if lower_bound_version > upper_bound_version:
|
|
upper_bound_str = None
|
|
if lower_bound_version == upper_bound_version:
|
|
if lower_bound_str[1] == "=" and upper_bound_str[1] == "=":
|
|
return pip.req.InstallRequirement.from_line(
|
|
"%s==%s" % (req_key, upper_bound_str[2:]),
|
|
"compiled")
|
|
else:
|
|
upper_bound_str = None
|
|
req_specs = []
|
|
req_key = req_list[0].req.key
|
|
if lower_bound_str:
|
|
req_specs.append(lower_bound_str)
|
|
if upper_bound_str:
|
|
req_specs.append(upper_bound_str)
|
|
req_specs.extend(conflicts)
|
|
return pip.req.InstallRequirement.from_line(
|
|
"%s%s" % (req_key, ",".join(req_specs)),
|
|
"compiled")
|
|
|
|
|
|
def join_requirements(options):
|
|
global joined_requirements
|
|
all_requirements, ignored_requirements = parse_requirements(options)
|
|
skip_keys = set(pkg.req.key for pkg in ignored_requirements)
|
|
installed_by_key = {}
|
|
installed_requrements = []
|
|
if options.ignore_installed or options.frozen:
|
|
installed_requrements = installed_packages(options)
|
|
if options.ignore_installed:
|
|
skip_keys |= set(pkg.key for pkg in installed_requrements)
|
|
if options.frozen:
|
|
installed_by_key = dict((pkg.key, pkg) for pkg in installed_requrements)
|
|
|
|
for req_key, req_list in all_requirements.iteritems():
|
|
if req_key in skip_keys:
|
|
continue
|
|
joined_req = join_one_requirement(req_list)
|
|
try:
|
|
installed_req = installed_by_key[req_key]
|
|
installed_version = installed_req.index[0][0]
|
|
except (KeyError, IndexError):
|
|
pass
|
|
else:
|
|
if installed_version not in joined_req.req:
|
|
frozen_req = pip.req.InstallRequirement.from_line(
|
|
"%s>=%s" % (installed_req.project_name,
|
|
installed_req.specs[0][1]),
|
|
"pip freeze")
|
|
incompatible_requirement(frozen_req, joined_req)
|
|
joined_req = frozen_req
|
|
joined_requirements.append(joined_req)
|
|
|
|
segment_ok = False
|
|
lower_version = None
|
|
lower_strict = False
|
|
exact_version = None
|
|
conflicts = []
|
|
for parsed, trans, op, ver in joined_req.req.index:
|
|
if op[0] == ">":
|
|
lower_version = parsed
|
|
lower_strict = len(op) == 2
|
|
elif op[0] == "<":
|
|
segment_ok = True
|
|
elif op[0] == "=":
|
|
exact_version = parsed
|
|
else:
|
|
conflicts.append(parsed)
|
|
if exact_version:
|
|
for req in req_list:
|
|
if not exact_version in req.req:
|
|
incompatible_requirement(joined_req, req)
|
|
else:
|
|
for req in req_list:
|
|
for parsed, trans, op, ver in req.req.index:
|
|
if op[0] == "=":
|
|
if parsed in conflicts:
|
|
incompatible_requirement(joined_req, req)
|
|
break
|
|
elif not segment_ok and op[0] == "<":
|
|
# analyse lower bound: x >= A or x > A
|
|
if (lower_version > parsed or (
|
|
lower_version == parsed and
|
|
(lower_strict or len(op) != 2))):
|
|
incompatible_requirement(joined_req, req)
|
|
break
|
|
|
|
|
|
def print_requirements():
|
|
formatted_requirements = []
|
|
for req in joined_requirements:
|
|
if req.url:
|
|
req = "%s#egg=%s" % (req.url, req.req)
|
|
else:
|
|
req = str(req.req)
|
|
formatted_requirements.append(req)
|
|
for req in sorted(formatted_requirements):
|
|
print req
|
|
|
|
|
|
def main():
|
|
parser = create_parser()
|
|
options = parser.parse_args()
|
|
setup_logging(options)
|
|
join_requirements(options)
|
|
print_requirements()
|
|
if incompatibles:
|
|
sys.exit(INCOMPATIBLE_REQUIREMENTS)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|