x/anvil
Code Issues Proposed changes

Use traditional Unix style for multipip

Browse Source 
multipip output can be parsed by pip tool. All errors
go to stderr. Anvil reports where incompatible requirements
were found.

Change-Id: Iba110e10ed3e2580d60ef475ac877190d1d03fcc
Fixes: bug #1188080
This commit is contained in:
Alessio Ababilov
2013-06-06 09:16:43 +04:00
committed by Joshua Harlow
parent 06fc741026
commit f016749dd0
3 changed files with 71 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
anvil/packaging/base.py
View File

@@ -18,11 +18,9 @@
# R0921: Abstract class not referenced
#pylint: disable=R0902,R0921
import json
import pkg_resources
from anvil import colorizer
from anvil import exceptions as excp
from anvil import log as logging
from anvil import shell as sh
from anvil import utils
@@ -151,7 +149,7 @@ class DependencyHandler(object):
req = pkg_resources.Requirement.parse(line)
new_lines.append(str(forced_by_key[req.key]))
except:
# We don't force the package or it has a bad format
# we don't force the package or it has a bad format
new_lines.append(line)
contents = "# Cleaned on %s\n\n%s\n" % (
utils.iso8601(), "\n".join(new_lines))
@@ -173,46 +171,34 @@ class DependencyHandler(object):
]
cmdline = cmdline + extra_pips + ["-r"] + requires_files
output = sh.execute(cmdline)
contents = json.loads(output[0])
# Figure out which ones were easily matched.
self.pips_to_install = []
for pkg in contents.get('compatibles', []):
if pkg.lower() not in OPENSTACK_PACKAGES:
self.pips_to_install.append(pkg)
# Figure out which ones we are forced to install.
output = sh.execute(cmdline, check_exit_code=False)
conflict_descr = output[1].strip()
forced_keys = set()
for (k, req_list) in contents.get('incompatibles', {}):
forced_keys.add(k.lower())
# Select the first.
if req_list and req_list[0] not in self.pips_to_install:
self.pips_to_install.append(req_list[0])
if conflict_descr:
for line in conflict_descr.splitlines():
LOG.warning(line)
if line.endswith(": incompatible requirements"):
forced_keys.add(line.split(":", 1)[0].lower())
self.pips_to_install = [
pkg
for pkg in utils.splitlines_not_empty(output[0])
if pkg.lower() not in OPENSTACK_PACKAGES]
sh.write_file(self.gathered_requires_filename,
"\n".join(self.pips_to_install))
if not self.pips_to_install:
LOG.error("No dependencies for OpenStack found. "
LOG.error("No dependencies for OpenStack found."
"Something went wrong. Please check:")
LOG.error("'%s'" % "' '".join(cmdline))
raise excp.AnvilException("No dependencies for OpenStack found!")
raise RuntimeError("No dependencies for OpenStack found")
utils.log_iterable(sorted(self.pips_to_install),
logger=LOG,
header="Python dependency list (all)")
sh.write_file(self.gathered_requires_filename,
"\n".join(self.pips_to_install))
header="Full known Python dependency list")
self.forced_packages = []
for k in forced_keys:
for pip in self.pips_to_install:
req = pkg_resources.Requirement.parse(pip)
if req.key == k:
self.forced_packages.append(req)
break
if self.forced_packages:
utils.log_iterable(sorted(self.forced_packages),
logger=LOG,
header="Python dependency list (forced)")
for pip in self.pips_to_install:
req = pkg_resources.Requirement.parse(pip)
if req.key in forced_keys:
self.forced_packages.append(req)
sh.write_file(self.forced_requires_filename,
"\n".join(str(req) for req in self.forced_packages))

27
tools/README.rst
View File

@@ -12,23 +12,34 @@ multipip
Use `multipip` to join these requirements::
$ multipip 'nose>=1.2' 'nose>=2' 'nose<4'
{"compatibles": ["nose>=2,<4"], "incompatibles": {}}
nose>=2,<4
`multipip` can be used to run `pip`::
$ pip install $(multipip -r pip-requires)
...
Files of requirements can be used as well::
$ cat pip-requires
nose<4
$ multipip 'nose>=1.2' 'nose>=2' -r pip-requires
{"compatibles": ["nose>=2,<4"], "incompatibles": {}}
nose>=2,<4
`multipip` prints error messages for badly formated requirements and exits early
and for incompatible requirements provides you which package was incompatible
and which versions were found to be problematic::
`multipip` prints error messages for incompatible requirements to
stderr and chooses the first one (note: command-line requirements take
precedence over files)::
$ cat pip-requires
pip==1.3
$ multipip 'pip==1.2' -r pip-requires
{"compatibles": [], "incompatibles": {"pip": ["pip==1.2", "pip==1.3"]}}
pip: incompatible requirements
Choosing:
command line: pip==1.2
Conflicting:
-r pip-requires (line 1): pip==1.3
pip==1.2
It is possible to filter some packages from printed output. This can
be useful for a huge `pip-requires` file::
@@ -38,7 +49,7 @@ be useful for a huge `pip-requires` file::
pip==1.2
nose>=1.2
$ multipip -r pip-requires --ignore-packages nose
{"compatibles": ["pip==1.2"], "incompatibles": {}}
pip==1.2
Installed packages can be filtered, too (they are taken from `pip
freeze`)::
@@ -50,7 +61,7 @@ freeze`)::
$ pip freeze | grep nose
nose==1.1.2
$ multipip -r pip-requires --ignore-installed
{"compatibles": ["pip==1.2"], "incompatibles": {}}
pip==1.2
py2rpm
------

71
tools/multipip
View File

@@ -1,9 +1,7 @@
#!/usr/bin/python
import argparse
import collections
import distutils.spawn
import json
import logging
import os
import subprocess
@@ -16,6 +14,7 @@ import pkg_resources
BAD_REQUIREMENTS = 2
INCOMPATIBLE_REQUIREMENTS = 3
logger = logging.getLogger()
@@ -85,7 +84,7 @@ def setup_logging(options):
logger.setLevel(level)
incompatibles = collections.defaultdict(list)
incompatibles = set()
joined_requirements = []
@@ -118,15 +117,18 @@ def install_requirement_parse(line, comes_from):
return install_requirement_ensure_req_field(req)
def add_incompatible_requirement(chosen, conflicting):
global incompatibles
key = chosen.req.key
if key not in incompatibles:
incompatibles[key].extend([install_requirement_str(conflicting),
install_requirement_str(chosen)])
else:
incompatibles[key].append(install_requirement_str(chosen))
def incompatible_requirement(chosen, conflicting):
if chosen.req.key not in incompatibles:
incompatibles.add(chosen.req.key)
print >> sys.stderr, "%s: incompatible requirements" % chosen.req.key
print >> sys.stderr, "Choosing:"
print >> sys.stderr, ("\t%s: %s" %
(chosen.comes_from,
install_requirement_str(chosen)))
print >> sys.stderr, "Conflicting:"
print >> sys.stderr, ("\t%s: %s" %
(conflicting.comes_from,
install_requirement_str(conflicting)))
def parse_requirements(options):
@@ -134,24 +136,22 @@ def parse_requirements(options):
:return: tuple (all, ignored) of InstallRequirement
"""
all_requirements = collections.defaultdict(list)
# CLI directly provided requirements first.
all_requirements = {}
for req_spec in options.requirement_specs:
try:
req = install_requirement_parse(req_spec, "command line")
all_requirements.setdefault(req.req.key, []).append(req)
except Exception as ex:
logger.error("Cannot parse `%s': %s" % (req_spec, ex))
sys.exit(BAD_REQUIREMENTS)
for filename in options.requirements:
try:
for req in pip.req.parse_requirements(filename, options=options):
req = install_requirement_ensure_req_field(req)
all_requirements[req.req.key].append(req)
all_requirements.setdefault(req.req.key, []).append(req)
except Exception as ex:
logger.error("Cannot parse `%s': %s" % (filename, ex))
sys.exit(BAD_REQUIREMENTS)
# CLI provided files after that.
for req_spec in options.requirement_specs:
try:
req = install_requirement_parse(req_spec, "command line")
all_requirements[req.req.key].append(req)
except Exception as ex:
logger.error("Cannot parse `%s': %s" % (req_spec, ex))
sys.exit(BAD_REQUIREMENTS)
ignored_requirements = []
for req_spec in options.ignore_packages:
try:
@@ -249,7 +249,6 @@ def join_one_requirement(req_list):
def join_requirements(options):
global joined_requirements
all_requirements, ignored_requirements = parse_requirements(options)
skip_keys = set(pkg.req.key for pkg in ignored_requirements)
installed_by_key = {}
@@ -276,7 +275,7 @@ def join_requirements(options):
"%s>=%s" % (installed_req.project_name,
installed_req.specs[0][1]),
"pip freeze")
add_incompatible_requirement(frozen_req, joined_req)
incompatible_requirement(frozen_req, joined_req)
joined_req = frozen_req
joined_requirements.append(joined_req.req)
@@ -298,36 +297,26 @@ def join_requirements(options):
if exact_version:
for req in req_list:
if not exact_version in req.req:
add_incompatible_requirement(joined_req, req)
incompatible_requirement(joined_req, req)
else:
for req in req_list:
for parsed, trans, op, ver in req.req.index:
if op[0] == "=":
if parsed in conflicts:
add_incompatible_requirement(joined_req, req)
incompatible_requirement(joined_req, req)
break
elif not segment_ok and op[0] == "<":
# analyse lower bound: x >= A or x > A
if (lower_version > parsed or (
lower_version == parsed and
(lower_strict or len(op) != 2))):
add_incompatible_requirement(joined_req, req)
incompatible_requirement(joined_req, req)
break
def print_requirements():
global joined_requirements
global incompatibles
requirements = {
'compatibles': [],
'incompatibles': {},
}
for r in joined_requirements:
if r.key not in incompatibles:
requirements['compatibles'].append(str(r))
requirements['incompatibles'].update(incompatibles)
print(json.dumps(requirements))
for req in sorted(joined_requirements, key=lambda x: x.key):
print req
def main():
@@ -336,6 +325,8 @@ def main():
setup_logging(options)
join_requirements(options)
print_requirements()
if incompatibles:
sys.exit(INCOMPATIBLE_REQUIREMENTS)
if __name__ == "__main__":