Use traditional Unix style for multipip

multipip output can be parsed by pip tool. All errors
go to stderr. Anvil reports where incompatible requirements
were found.

Change-Id: Iba110e10ed3e2580d60ef475ac877190d1d03fcc
Fixes: bug #1188080

anvil/packaging/base.py

@@ -18,11 +18,9 @@
 # R0921: Abstract class not referenced
 #pylint: disable=R0902,R0921
 
-import json
 import pkg_resources
 
 from anvil import colorizer
-from anvil import exceptions as excp
 from anvil import log as logging
 from anvil import shell as sh
 from anvil import utils
@@ -151,7 +149,7 @@ class DependencyHandler(object):
                     req = pkg_resources.Requirement.parse(line)
                     new_lines.append(str(forced_by_key[req.key]))
                 except:
-                    # We don't force the package or it has a bad format
+                    # we don't force the package or it has a bad format
                     new_lines.append(line)
             contents = "# Cleaned on %s\n\n%s\n" % (
                 utils.iso8601(), "\n".join(new_lines))
@@ -173,46 +171,34 @@ class DependencyHandler(object):
         ]
         cmdline = cmdline + extra_pips + ["-r"] + requires_files
 
-        output = sh.execute(cmdline)
+        output = sh.execute(cmdline, check_exit_code=False)
-        contents = json.loads(output[0])
+        conflict_descr = output[1].strip()
-
-        # Figure out which ones were easily matched.
-        self.pips_to_install = []
-        for pkg in contents.get('compatibles', []):
-            if pkg.lower() not in OPENSTACK_PACKAGES:
-                self.pips_to_install.append(pkg)
-
-        # Figure out which ones we are forced to install.
         forced_keys = set()
-        for (k, req_list) in contents.get('incompatibles', {}):
+        if conflict_descr:
-            forced_keys.add(k.lower())
+            for line in conflict_descr.splitlines():
-            # Select the first.
+                LOG.warning(line)
-            if req_list and req_list[0] not in self.pips_to_install:
+                if line.endswith(": incompatible requirements"):
-                self.pips_to_install.append(req_list[0])
+                    forced_keys.add(line.split(":", 1)[0].lower())
-
+        self.pips_to_install = [
+            pkg
+            for pkg in utils.splitlines_not_empty(output[0])
+            if pkg.lower() not in OPENSTACK_PACKAGES]
+        sh.write_file(self.gathered_requires_filename,
+                      "\n".join(self.pips_to_install))
         if not self.pips_to_install:
-            LOG.error("No dependencies for OpenStack found. "
+            LOG.error("No dependencies for OpenStack found."
                       "Something went wrong. Please check:")
             LOG.error("'%s'" % "' '".join(cmdline))
-            raise excp.AnvilException("No dependencies for OpenStack found!")
+            raise RuntimeError("No dependencies for OpenStack found")
 
         utils.log_iterable(sorted(self.pips_to_install),
                            logger=LOG,
-                           header="Python dependency list (all)")
+                           header="Full known Python dependency list")
-        sh.write_file(self.gathered_requires_filename,
-                      "\n".join(self.pips_to_install))
-
         self.forced_packages = []
-        for k in forced_keys:
+        for pip in self.pips_to_install:
-            for pip in self.pips_to_install:
+            req = pkg_resources.Requirement.parse(pip)
-                req = pkg_resources.Requirement.parse(pip)
+            if req.key in forced_keys:
-                if req.key == k:
+                self.forced_packages.append(req)
-                    self.forced_packages.append(req)
-                    break
-        if self.forced_packages:
-            utils.log_iterable(sorted(self.forced_packages),
-                               logger=LOG,
-                               header="Python dependency list (forced)")
         sh.write_file(self.forced_requires_filename,
                       "\n".join(str(req) for req in self.forced_packages))
 

tools/README.rst

@@ -12,23 +12,34 @@ multipip
 Use `multipip` to join these requirements::
 
     $ multipip 'nose>=1.2' 'nose>=2' 'nose<4'
-    {"compatibles": ["nose>=2,<4"], "incompatibles": {}}
+    nose>=2,<4
+
+
+`multipip` can be used to run `pip`::
+
+   $ pip install $(multipip -r pip-requires)
+   ...
 
 Files of requirements can be used as well::
 
     $ cat pip-requires
     nose<4
     $ multipip 'nose>=1.2' 'nose>=2' -r pip-requires
-    {"compatibles": ["nose>=2,<4"], "incompatibles": {}}
+    nose>=2,<4
 
-`multipip` prints error messages for badly formated requirements and exits early
+`multipip` prints error messages for incompatible requirements to
-and for incompatible requirements provides you which package was incompatible
+stderr and chooses the first one (note: command-line requirements take
-and which versions were found to be problematic::
+precedence over files)::
 
     $ cat pip-requires
     pip==1.3
     $ multipip 'pip==1.2' -r pip-requires
-    {"compatibles": [], "incompatibles": {"pip": ["pip==1.2", "pip==1.3"]}}
+    pip: incompatible requirements
+    Choosing:
+    	command line: pip==1.2
+    Conflicting:
+    	-r pip-requires (line 1): pip==1.3
+    pip==1.2
 
 It is possible to filter some packages from printed output. This can
 be useful for a huge `pip-requires` file::
@@ -38,7 +49,7 @@ be useful for a huge `pip-requires` file::
     pip==1.2
     nose>=1.2
     $ multipip -r pip-requires --ignore-packages nose
-    {"compatibles": ["pip==1.2"], "incompatibles": {}}
+    pip==1.2
 
 Installed packages can be filtered, too (they are taken from `pip
 freeze`)::
@@ -50,7 +61,7 @@ freeze`)::
     $ pip freeze | grep nose
     nose==1.1.2
     $ multipip -r pip-requires --ignore-installed
-    {"compatibles": ["pip==1.2"], "incompatibles": {}}
+    pip==1.2
 
 py2rpm
 ------

tools/multipip

@@ -1,9 +1,7 @@
 #!/usr/bin/python
 
 import argparse
-import collections
 import distutils.spawn
-import json
 import logging
 import os
 import subprocess
@@ -16,6 +14,7 @@ import pkg_resources
 
 
 BAD_REQUIREMENTS = 2
+INCOMPATIBLE_REQUIREMENTS = 3
 logger = logging.getLogger()
 
 
@@ -85,7 +84,7 @@ def setup_logging(options):
     logger.setLevel(level)
 
 
-incompatibles = collections.defaultdict(list)
+incompatibles = set()
 joined_requirements = []
 
 
@@ -118,15 +117,18 @@ def install_requirement_parse(line, comes_from):
     return install_requirement_ensure_req_field(req)
 
 
-def add_incompatible_requirement(chosen, conflicting):
+def incompatible_requirement(chosen, conflicting):
-    global incompatibles
+    if chosen.req.key not in incompatibles:
-
+        incompatibles.add(chosen.req.key)
-    key = chosen.req.key
+        print >> sys.stderr, "%s: incompatible requirements" % chosen.req.key
-    if key not in incompatibles:
+        print >> sys.stderr, "Choosing:"
-        incompatibles[key].extend([install_requirement_str(conflicting),
+        print >> sys.stderr, ("\t%s: %s" %
-                                   install_requirement_str(chosen)])
+                              (chosen.comes_from,
-    else:
+                               install_requirement_str(chosen)))
-        incompatibles[key].append(install_requirement_str(chosen))
+        print >> sys.stderr, "Conflicting:"
+    print >> sys.stderr, ("\t%s: %s" %
+                          (conflicting.comes_from,
+                           install_requirement_str(conflicting)))
 
 
 def parse_requirements(options):
@@ -134,24 +136,22 @@ def parse_requirements(options):
 
     :return: tuple (all, ignored) of InstallRequirement
     """
-    all_requirements = collections.defaultdict(list)
+    all_requirements = {}
-    # CLI directly provided requirements first.
+    for req_spec in options.requirement_specs:
+        try:
+            req = install_requirement_parse(req_spec, "command line")
+            all_requirements.setdefault(req.req.key, []).append(req)
+        except Exception as ex:
+            logger.error("Cannot parse `%s': %s" % (req_spec, ex))
+            sys.exit(BAD_REQUIREMENTS)
     for filename in options.requirements:
         try:
             for req in pip.req.parse_requirements(filename, options=options):
                 req = install_requirement_ensure_req_field(req)
-                all_requirements[req.req.key].append(req)
+                all_requirements.setdefault(req.req.key, []).append(req)
         except Exception as ex:
             logger.error("Cannot parse `%s': %s" % (filename, ex))
             sys.exit(BAD_REQUIREMENTS)
-    # CLI provided files after that.
-    for req_spec in options.requirement_specs:
-        try:
-            req = install_requirement_parse(req_spec, "command line")
-            all_requirements[req.req.key].append(req)
-        except Exception as ex:
-            logger.error("Cannot parse `%s': %s" % (req_spec, ex))
-            sys.exit(BAD_REQUIREMENTS)
     ignored_requirements = []
     for req_spec in options.ignore_packages:
         try:
@@ -249,7 +249,6 @@ def join_one_requirement(req_list):
 
 def join_requirements(options):
     global joined_requirements
-
     all_requirements, ignored_requirements = parse_requirements(options)
     skip_keys = set(pkg.req.key for pkg in ignored_requirements)
     installed_by_key = {}
@@ -276,7 +275,7 @@ def join_requirements(options):
                     "%s>=%s" % (installed_req.project_name,
                                 installed_req.specs[0][1]),
                     "pip freeze")
-                add_incompatible_requirement(frozen_req, joined_req)
+                incompatible_requirement(frozen_req, joined_req)
                 joined_req = frozen_req
         joined_requirements.append(joined_req.req)
 
@@ -298,36 +297,26 @@ def join_requirements(options):
         if exact_version:
             for req in req_list:
                 if not exact_version in req.req:
-                    add_incompatible_requirement(joined_req, req)
+                    incompatible_requirement(joined_req, req)
         else:
             for req in req_list:
                 for parsed, trans, op, ver in req.req.index:
                     if op[0] == "=":
                         if parsed in conflicts:
-                            add_incompatible_requirement(joined_req, req)
+                            incompatible_requirement(joined_req, req)
                             break
                     elif not segment_ok and op[0] == "<":
                         # analyse lower bound: x >= A or x > A
                         if (lower_version > parsed or (
                                 lower_version == parsed and
                                 (lower_strict or len(op) != 2))):
-                            add_incompatible_requirement(joined_req, req)
+                            incompatible_requirement(joined_req, req)
                             break
 
 
 def print_requirements():
-    global joined_requirements
+    for req in sorted(joined_requirements, key=lambda x: x.key):
-    global incompatibles
+        print req
-
-    requirements = {
-        'compatibles': [],
-        'incompatibles': {},
-    }
-    for r in joined_requirements:
-        if r.key not in incompatibles:
-            requirements['compatibles'].append(str(r))
-    requirements['incompatibles'].update(incompatibles)
-    print(json.dumps(requirements))
 
 
 def main():
@@ -336,6 +325,8 @@ def main():
     setup_logging(options)
     join_requirements(options)
     print_requirements()
+    if incompatibles:
+        sys.exit(INCOMPATIBLE_REQUIREMENTS)
 
 
 if __name__ == "__main__":