When a requirement url is used don't lose it.

Previously it seems like we lost requirement
urls and then downloading said requirment url
would fail (since the url is typically provided
if the package isn't on pypi yet). This seems
to be happening for the latests nova requirement
which seems to try to pull in a special oslo
config version.

Change-Id: I30acea47f07f6d189fd63ab9e90434f1eb4e4e2d

anvil/packaging/base.py

@@ -180,7 +180,7 @@ class DependencyHandler(object):
             new_lines = []
             for line in old_lines:
                 try:
-                    req = pkg_resources.Requirement.parse(line)
+                    req = pip_helper.extract_requirement(line)
                     new_lines.append(str(forced_by_key[req.key]))
                 except:
                     # we don't force the package or it has a bad format
@@ -213,10 +213,11 @@ class DependencyHandler(object):
                 LOG.warning(line)
                 if line.endswith(": incompatible requirements"):
                     forced_keys.add(line.split(":", 1)[0].lower())
-        self.pips_to_install = [
+        self.pips_to_install = []
-            pkg
+        for line in utils.splitlines_not_empty(output[0]):
-            for pkg in utils.splitlines_not_empty(output[0])
+            req = pip_helper.extract_requirement(line)
-            if pkg.lower() not in OPENSTACK_PACKAGES]
+            if req.key not in OPENSTACK_PACKAGES:
+                self.pips_to_install.append(line)
         sh.write_file(self.gathered_requires_filename,
                       "\n".join(self.pips_to_install))
         if not self.pips_to_install:
@@ -229,8 +230,8 @@ class DependencyHandler(object):
                            logger=LOG,
                            header="Full known python dependency list")
         self.forced_packages = []
-        for pip in self.pips_to_install:
+        for line in self.pips_to_install:
-            req = pkg_resources.Requirement.parse(pip)
+            req = pip_helper.extract_requirement(line)
             if req.key in forced_keys:
                 self.forced_packages.append(req)
         sh.write_file(self.forced_requires_filename,
@@ -290,11 +291,9 @@ class DependencyHandler(object):
         download_requires_filename = sh.joinpths(self.deps_dir,
                                                  "download-requires")
         raw_pips_to_download = self.filter_download_requires()
-        pips_to_download = [pkg_resources.Requirement.parse(str(p.strip()))
-                            for p in raw_pips_to_download if p.strip()]
         sh.write_file(download_requires_filename,
-                      "\n".join(str(req) for req in pips_to_download))
+                      "\n".join(str(req) for req in raw_pips_to_download))
-        if not pips_to_download:
+        if not raw_pips_to_download:
             return ([], [])
         pip_dir = sh.joinpths(self.deps_dir, "pip")
         pip_download_dir = sh.joinpths(pip_dir, "download")
@@ -303,19 +302,19 @@ class DependencyHandler(object):
         if clear_cache:
             sh.deldir(pip_cache_dir)
         pip_failures = []
-        how_many = len(pips_to_download)
         for attempt in xrange(self.MAX_PIP_DOWNLOAD_ATTEMPTS):
             # NOTE(aababilov): pip has issues with already downloaded files
             sh.deldir(pip_download_dir)
             sh.mkdir(pip_download_dir, recurse=True)
             sh.deldir(pip_build_dir)
+            header = "Downloading %s python dependencies (attempt %s)"
+            header = header % (len(raw_pips_to_download), attempt)
             utils.log_iterable(sorted(raw_pips_to_download),
                                logger=LOG,
-                               header=("Downloading %s python dependencies "
+                               header=header)
-                                       "(attempt %s)" % (how_many, attempt)))
             failed = False
             try:
-                self._try_download_dependencies(attempt, pips_to_download,
+                self._try_download_dependencies(attempt, raw_pips_to_download,
                                                 pip_download_dir,
                                                 pip_cache_dir, pip_build_dir)
                 pip_failures = []
@@ -327,8 +326,10 @@ class DependencyHandler(object):
                 break
         if pip_failures:
             raise pip_failures[-1]
-        self._examine_download_dir(pips_to_download, pip_download_dir)
+        pips_downloaded = [pip_helper.extract_requirement(p)
+                           for p in raw_pips_to_download]
+        self._examine_download_dir(pips_downloaded, pip_download_dir)
         for filename in sh.listdir(pip_download_dir, files_only=True):
             sh.move(filename, self.download_dir)
         what_downloaded = sh.listdir(self.download_dir, files_only=True)
-        return (pips_to_download, what_downloaded)
+        return (pips_downloaded, what_downloaded)

anvil/packaging/helpers/pip_helper.py

@@ -47,6 +47,15 @@ def create_requirement(name, version=None):
     return pkg_resources.Requirement.parse(name)
 
 
+def extract(line):
+    return pip_req.InstallRequirement.from_line(line)
+
+
+def extract_requirement(line):
+    req = extract(line)
+    return req.req
+
+
 def get_directory_details(path):
     if not sh.isdir(path):
         raise IOError("Can not detail non-existent directory %s" % (path))
@@ -57,7 +66,7 @@ def get_directory_details(path):
     if cache_key in EGGS_DETAILED:
         return EGGS_DETAILED[cache_key]
 
-    req = pip_req.InstallRequirement.from_line(path)
+    req = extract(path)
     req.source_dir = path
     req.run_egg_info()
 

anvil/packaging/yum.py

@@ -176,14 +176,19 @@ class YumDependencyHandler(base.DependencyHandler):
 
     def filter_download_requires(self):
         yum_map = self._get_yum_available()
-        nopips = [pkg_resources.Requirement.parse(name).key
+        no_pips = [pkg_resources.Requirement.parse(name).key
                   for name in self.python_names]
 
+        pip_origins = {}
+        for line in self.pips_to_install:
+            req = pip_helper.extract_requirement(line)
+            pip_origins[req.key] = line
+
         pips_to_download = []
-        req_to_install = [pkg_resources.Requirement.parse(pkg)
+        req_to_install = [pip_helper.extract_requirement(line)
-                          for pkg in self.pips_to_install]
+                          for line in self.pips_to_install]
         req_to_install = [req for req in req_to_install
-                          if req.key not in nopips]
+                          if req.key not in no_pips]
 
         requested_names = [req.key for req in req_to_install]
         rpm_to_install = self._convert_names_python2rpm(requested_names)
@@ -192,7 +197,7 @@ class YumDependencyHandler(base.DependencyHandler):
         for (req, rpm_name) in zip(req_to_install, rpm_to_install):
             (version, repo) = self._find_yum_match(yum_map, req, rpm_name)
             if not repo:
-                pips_to_download.append(str(req))
+                pips_to_download.append(pip_origins[req.key])
             else:
                 satisfied_list.append((req, rpm_name, version, repo))
 

tools/multipip

@@ -285,7 +285,7 @@ def join_requirements(options):
                     "pip freeze")
                 incompatible_requirement(frozen_req, joined_req)
                 joined_req = frozen_req
-        joined_requirements.append(joined_req.req)
+        joined_requirements.append(joined_req)
 
         segment_ok = False
         lower_version = None
@@ -323,7 +323,14 @@ def join_requirements(options):
 
 
 def print_requirements():
-    for req in sorted(joined_requirements, key=lambda x: x.key):
+    formatted_requirements = []
+    for req in joined_requirements:
+        if req.url:
+            req = "%s#egg=%s" % (req.url, req.req)
+        else:
+            req = str(req.req)
+        formatted_requirements.append(req)
+    for req in sorted(formatted_requirements):
         print req