Fix up editables in multipip, pip_helper

Currently we aren't processing editables correctly
through the various pipelines (multiple file resolution,
analysis, downloading, packaging) so this starts to fix
some of these issues to make sure they do flow through
the pipeline correctly.

Change-Id: I5fa40cadd41a657d8f8824d6fd8df45efbd00872

anvil/components/base_install.py

@@ -141,9 +141,8 @@ class PythonComponent(base.BasicComponent):
     @property
     def egg_info(self):
         egg_info = pip_helper.get_directory_details(self.get_option('app_dir')).copy()
-        read_reqs = pip_helper.read_requirement_files
-        egg_info['dependencies'] = read_reqs(self.requires_files)
-        egg_info['test_dependencies'] = read_reqs(self.test_requires_files)
+        egg_info['dependencies'] = pip_helper.read_requirement_files(self.requires_files)[1]
+        egg_info['test_dependencies'] = pip_helper.read_requirement_files(self.test_requires_files)[1]
         return egg_info
 
 

anvil/packaging/base.py

@@ -261,8 +261,8 @@ class DependencyHandler(object):
                                                            forced_pips)
         self.pips_to_install = compatibles
         sh.write_file(self.gathered_requires_filename, "\n".join(self.pips_to_install))
-        pips_to_install = pip_helper.read_requirement_files([self.gathered_requires_filename])
-        pips_to_install = sorted(pips_to_install, cmp=sort_req)
+        pip_requirements, raw_requirements = pip_helper.read_requirement_files([self.gathered_requires_filename])
+        pips_to_install = sorted(raw_requirements, cmp=sort_req)
         utils.log_iterable(pips_to_install, logger=LOG,
                            header="Full known python dependency list")
 

anvil/packaging/helpers/pip_helper.py

@@ -17,6 +17,7 @@
 from distutils import version as dist_version
 import pkg_resources
 import re
+import tempfile
 
 from pip import req as pip_req
 
@@ -36,8 +37,6 @@ EGGS_DETAILED = {}
 PYTHON_KEY_VERSION_RE = re.compile("^(.+)-([0-9][0-9.a-zA-Z]*)$")
 PIP_VERSION = pkg_resources.get_distribution('pip').version
 PIP_EXECUTABLE = sh.which_first(['pip', 'pip-python'])
-OPENSTACK_TARBALLS_RE = re.compile(r'http://tarballs.openstack.org/([^/]+)/')
-SKIP_LINES = ('#', '-e', '-f', 'http://', 'https://')
 
 
 def create_requirement(name, version=None):
@@ -57,8 +56,26 @@ def create_requirement(name, version=None):
     return pkg_resources.Requirement.parse(name)
 
 
+def _split(line):
+    if line.startswith('-e') or line.startswith('--editable'):
+        if line.startswith('-e'):
+            line = line[2:].strip()
+        else:
+            line = line[len('--editable'):].strip().lstrip('=')
+        if line:
+            return ('-e', line)
+    return (None, line)
+
+
 def extract(line):
-    req = pip_req.InstallRequirement.from_line(line)
+    if line.startswith('-e') or line.startswith('--editable'):
+        if line.startswith('-e'):
+            line = line[2:].strip()
+        else:
+            line = line[len('--editable'):].strip().lstrip('=')
+        req = pip_req.InstallRequirement.from_editable(line, comes_from="??")
+    else:
+        req = pip_req.InstallRequirement.from_line(line, comes_from="??")
     # NOTE(aababilov): req.req.key can look like oslo.config-1.2.0a2,
     # so, split it
     if req.req:
@@ -133,31 +150,22 @@ def get_archive_details(filename):
     return details
 
 
-def _skip_requirement(line):
-    return not len(line) or any(line.startswith(a) for a in SKIP_LINES)
-
-
-def parse_requirements(contents, adjust=False):
-    lines = []
-    for line in contents.splitlines():
-        line = line.strip()
-        if 'http://' in line:
-            m = OPENSTACK_TARBALLS_RE.search(line)
-            if m:
-                line = m.group(1)
-        if not _skip_requirement(line):
-            lines.append(line)
-    return pkg_resources.parse_requirements(lines)
+def parse_requirements(contents):
+    with tempfile.NamedTemporaryFile(suffix=".txt") as tmp_fh:
+        tmp_fh.write(contents)
+        tmp_fh.write("\n")
+        tmp_fh.flush()
+        return read_requirement_files([tmp_fh.name])
 
 
 def read_requirement_files(files):
-    result = []
+    pip_requirements = []
     for filename in files:
         if sh.isfile(filename):
             LOG.debug('Parsing requirements from %s', filename)
-            with open(filename) as f:
-                result.extend(parse_requirements(f.read()))
-    return result
+            pip_requirements.extend(pip_req.parse_requirements(filename))
+    return (pip_requirements,
+            [req.req for req in pip_requirements])
 
 
 def download_dependencies(download_dir, pips_to_download, output_filename):
@@ -191,5 +199,8 @@ def download_dependencies(download_dir, pips_to_download, output_filename):
     # See: https://github.com/pypa/pip/issues/1439
     if dist_version.StrictVersion(PIP_VERSION) >= dist_version.StrictVersion('1.5'):
         cmdline.append("--no-use-wheel")
-    cmdline.extend([str(p) for p in pips_to_download])
+    for p in pips_to_download:
+        for p_seg in _split(p):
+            if p_seg:
+                cmdline.append(p_seg)
     sh.execute_save_output(cmdline, output_filename)

anvil/packaging/yum.py

@@ -159,7 +159,7 @@ class YumDependencyHandler(base.DependencyHandler):
         # are not useful and should not be set in the first place).
         try:
             raw_downloaded = sh.load_file(self.build_requires_filename)
-            downloaded_reqs = pip_helper.parse_requirements(raw_downloaded)
+            _pip_reqs, downloaded_reqs = pip_helper.parse_requirements(raw_downloaded)
         except IOError as e:
             if e.errno != errno.ENOENT:
                 raise

tools/multipip

@@ -422,11 +422,14 @@ def print_requirements(joined_requirements):
     formatted_requirements = []
     for req_key in sorted(six.iterkeys(joined_requirements)):
         req = joined_requirements[req_key][0]
+        req_prefix = ""
+        if req.editable:
+            req_prefix = "-e "
         if req.url:
             req = "%s#egg=%s" % (req.url, req.req)
         else:
             req = str(req.req)
-        formatted_requirements.append(req)
+        formatted_requirements.append("%s%s" % (req_prefix, req))
     for req in formatted_requirements:
         print(req)