From 8806ac2a53e61ca42df192be8215c1766cf7ad12 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 19 Jun 2013 21:53:15 -0700 Subject: [PATCH] When a requirement url is used don't lose it. Previously it seems like we lost requirement urls and then downloading said requirment url would fail (since the url is typically provided if the package isn't on pypi yet). This seems to be happening for the latests nova requirement which seems to try to pull in a special oslo config version. Change-Id: I30acea47f07f6d189fd63ab9e90434f1eb4e4e2d --- anvil/packaging/base.py | 35 ++++++++++++++------------- anvil/packaging/helpers/pip_helper.py | 11 ++++++++- anvil/packaging/yum.py | 15 ++++++++---- tools/multipip | 11 +++++++-- 4 files changed, 47 insertions(+), 25 deletions(-) diff --git a/anvil/packaging/base.py b/anvil/packaging/base.py index 2be03219..2618db11 100644 --- a/anvil/packaging/base.py +++ b/anvil/packaging/base.py @@ -180,7 +180,7 @@ class DependencyHandler(object): new_lines = [] for line in old_lines: try: - req = pkg_resources.Requirement.parse(line) + req = pip_helper.extract_requirement(line) new_lines.append(str(forced_by_key[req.key])) except: # we don't force the package or it has a bad format @@ -213,10 +213,11 @@ class DependencyHandler(object): LOG.warning(line) if line.endswith(": incompatible requirements"): forced_keys.add(line.split(":", 1)[0].lower()) - self.pips_to_install = [ - pkg - for pkg in utils.splitlines_not_empty(output[0]) - if pkg.lower() not in OPENSTACK_PACKAGES] + self.pips_to_install = [] + for line in utils.splitlines_not_empty(output[0]): + req = pip_helper.extract_requirement(line) + if req.key not in OPENSTACK_PACKAGES: + self.pips_to_install.append(line) sh.write_file(self.gathered_requires_filename, "\n".join(self.pips_to_install)) if not self.pips_to_install: @@ -229,8 +230,8 @@ class DependencyHandler(object): logger=LOG, header="Full known python dependency list") self.forced_packages = [] - for pip in self.pips_to_install: - req = pkg_resources.Requirement.parse(pip) + for line in self.pips_to_install: + req = pip_helper.extract_requirement(line) if req.key in forced_keys: self.forced_packages.append(req) sh.write_file(self.forced_requires_filename, @@ -290,11 +291,9 @@ class DependencyHandler(object): download_requires_filename = sh.joinpths(self.deps_dir, "download-requires") raw_pips_to_download = self.filter_download_requires() - pips_to_download = [pkg_resources.Requirement.parse(str(p.strip())) - for p in raw_pips_to_download if p.strip()] sh.write_file(download_requires_filename, - "\n".join(str(req) for req in pips_to_download)) - if not pips_to_download: + "\n".join(str(req) for req in raw_pips_to_download)) + if not raw_pips_to_download: return ([], []) pip_dir = sh.joinpths(self.deps_dir, "pip") pip_download_dir = sh.joinpths(pip_dir, "download") @@ -303,19 +302,19 @@ class DependencyHandler(object): if clear_cache: sh.deldir(pip_cache_dir) pip_failures = [] - how_many = len(pips_to_download) for attempt in xrange(self.MAX_PIP_DOWNLOAD_ATTEMPTS): # NOTE(aababilov): pip has issues with already downloaded files sh.deldir(pip_download_dir) sh.mkdir(pip_download_dir, recurse=True) sh.deldir(pip_build_dir) + header = "Downloading %s python dependencies (attempt %s)" + header = header % (len(raw_pips_to_download), attempt) utils.log_iterable(sorted(raw_pips_to_download), logger=LOG, - header=("Downloading %s python dependencies " - "(attempt %s)" % (how_many, attempt))) + header=header) failed = False try: - self._try_download_dependencies(attempt, pips_to_download, + self._try_download_dependencies(attempt, raw_pips_to_download, pip_download_dir, pip_cache_dir, pip_build_dir) pip_failures = [] @@ -327,8 +326,10 @@ class DependencyHandler(object): break if pip_failures: raise pip_failures[-1] - self._examine_download_dir(pips_to_download, pip_download_dir) + pips_downloaded = [pip_helper.extract_requirement(p) + for p in raw_pips_to_download] + self._examine_download_dir(pips_downloaded, pip_download_dir) for filename in sh.listdir(pip_download_dir, files_only=True): sh.move(filename, self.download_dir) what_downloaded = sh.listdir(self.download_dir, files_only=True) - return (pips_to_download, what_downloaded) + return (pips_downloaded, what_downloaded) diff --git a/anvil/packaging/helpers/pip_helper.py b/anvil/packaging/helpers/pip_helper.py index a2e7a7bc..16184936 100644 --- a/anvil/packaging/helpers/pip_helper.py +++ b/anvil/packaging/helpers/pip_helper.py @@ -47,6 +47,15 @@ def create_requirement(name, version=None): return pkg_resources.Requirement.parse(name) +def extract(line): + return pip_req.InstallRequirement.from_line(line) + + +def extract_requirement(line): + req = extract(line) + return req.req + + def get_directory_details(path): if not sh.isdir(path): raise IOError("Can not detail non-existent directory %s" % (path)) @@ -57,7 +66,7 @@ def get_directory_details(path): if cache_key in EGGS_DETAILED: return EGGS_DETAILED[cache_key] - req = pip_req.InstallRequirement.from_line(path) + req = extract(path) req.source_dir = path req.run_egg_info() diff --git a/anvil/packaging/yum.py b/anvil/packaging/yum.py index 7e70363f..a6a9e6c7 100644 --- a/anvil/packaging/yum.py +++ b/anvil/packaging/yum.py @@ -176,14 +176,19 @@ class YumDependencyHandler(base.DependencyHandler): def filter_download_requires(self): yum_map = self._get_yum_available() - nopips = [pkg_resources.Requirement.parse(name).key + no_pips = [pkg_resources.Requirement.parse(name).key for name in self.python_names] + pip_origins = {} + for line in self.pips_to_install: + req = pip_helper.extract_requirement(line) + pip_origins[req.key] = line + pips_to_download = [] - req_to_install = [pkg_resources.Requirement.parse(pkg) - for pkg in self.pips_to_install] + req_to_install = [pip_helper.extract_requirement(line) + for line in self.pips_to_install] req_to_install = [req for req in req_to_install - if req.key not in nopips] + if req.key not in no_pips] requested_names = [req.key for req in req_to_install] rpm_to_install = self._convert_names_python2rpm(requested_names) @@ -192,7 +197,7 @@ class YumDependencyHandler(base.DependencyHandler): for (req, rpm_name) in zip(req_to_install, rpm_to_install): (version, repo) = self._find_yum_match(yum_map, req, rpm_name) if not repo: - pips_to_download.append(str(req)) + pips_to_download.append(pip_origins[req.key]) else: satisfied_list.append((req, rpm_name, version, repo)) diff --git a/tools/multipip b/tools/multipip index 251fcb50..81d82dc9 100755 --- a/tools/multipip +++ b/tools/multipip @@ -285,7 +285,7 @@ def join_requirements(options): "pip freeze") incompatible_requirement(frozen_req, joined_req) joined_req = frozen_req - joined_requirements.append(joined_req.req) + joined_requirements.append(joined_req) segment_ok = False lower_version = None @@ -323,7 +323,14 @@ def join_requirements(options): def print_requirements(): - for req in sorted(joined_requirements, key=lambda x: x.key): + formatted_requirements = [] + for req in joined_requirements: + if req.url: + req = "%s#egg=%s" % (req.url, req.req) + else: + req = str(req.req) + formatted_requirements.append(req) + for req in sorted(formatted_requirements): print req