diff --git a/anvil/packaging/base.py b/anvil/packaging/base.py index 2be03219..2618db11 100644 --- a/anvil/packaging/base.py +++ b/anvil/packaging/base.py @@ -180,7 +180,7 @@ class DependencyHandler(object): new_lines = [] for line in old_lines: try: - req = pkg_resources.Requirement.parse(line) + req = pip_helper.extract_requirement(line) new_lines.append(str(forced_by_key[req.key])) except: # we don't force the package or it has a bad format @@ -213,10 +213,11 @@ class DependencyHandler(object): LOG.warning(line) if line.endswith(": incompatible requirements"): forced_keys.add(line.split(":", 1)[0].lower()) - self.pips_to_install = [ - pkg - for pkg in utils.splitlines_not_empty(output[0]) - if pkg.lower() not in OPENSTACK_PACKAGES] + self.pips_to_install = [] + for line in utils.splitlines_not_empty(output[0]): + req = pip_helper.extract_requirement(line) + if req.key not in OPENSTACK_PACKAGES: + self.pips_to_install.append(line) sh.write_file(self.gathered_requires_filename, "\n".join(self.pips_to_install)) if not self.pips_to_install: @@ -229,8 +230,8 @@ class DependencyHandler(object): logger=LOG, header="Full known python dependency list") self.forced_packages = [] - for pip in self.pips_to_install: - req = pkg_resources.Requirement.parse(pip) + for line in self.pips_to_install: + req = pip_helper.extract_requirement(line) if req.key in forced_keys: self.forced_packages.append(req) sh.write_file(self.forced_requires_filename, @@ -290,11 +291,9 @@ class DependencyHandler(object): download_requires_filename = sh.joinpths(self.deps_dir, "download-requires") raw_pips_to_download = self.filter_download_requires() - pips_to_download = [pkg_resources.Requirement.parse(str(p.strip())) - for p in raw_pips_to_download if p.strip()] sh.write_file(download_requires_filename, - "\n".join(str(req) for req in pips_to_download)) - if not pips_to_download: + "\n".join(str(req) for req in raw_pips_to_download)) + if not raw_pips_to_download: return ([], []) pip_dir = sh.joinpths(self.deps_dir, "pip") pip_download_dir = sh.joinpths(pip_dir, "download") @@ -303,19 +302,19 @@ class DependencyHandler(object): if clear_cache: sh.deldir(pip_cache_dir) pip_failures = [] - how_many = len(pips_to_download) for attempt in xrange(self.MAX_PIP_DOWNLOAD_ATTEMPTS): # NOTE(aababilov): pip has issues with already downloaded files sh.deldir(pip_download_dir) sh.mkdir(pip_download_dir, recurse=True) sh.deldir(pip_build_dir) + header = "Downloading %s python dependencies (attempt %s)" + header = header % (len(raw_pips_to_download), attempt) utils.log_iterable(sorted(raw_pips_to_download), logger=LOG, - header=("Downloading %s python dependencies " - "(attempt %s)" % (how_many, attempt))) + header=header) failed = False try: - self._try_download_dependencies(attempt, pips_to_download, + self._try_download_dependencies(attempt, raw_pips_to_download, pip_download_dir, pip_cache_dir, pip_build_dir) pip_failures = [] @@ -327,8 +326,10 @@ class DependencyHandler(object): break if pip_failures: raise pip_failures[-1] - self._examine_download_dir(pips_to_download, pip_download_dir) + pips_downloaded = [pip_helper.extract_requirement(p) + for p in raw_pips_to_download] + self._examine_download_dir(pips_downloaded, pip_download_dir) for filename in sh.listdir(pip_download_dir, files_only=True): sh.move(filename, self.download_dir) what_downloaded = sh.listdir(self.download_dir, files_only=True) - return (pips_to_download, what_downloaded) + return (pips_downloaded, what_downloaded) diff --git a/anvil/packaging/helpers/pip_helper.py b/anvil/packaging/helpers/pip_helper.py index a2e7a7bc..16184936 100644 --- a/anvil/packaging/helpers/pip_helper.py +++ b/anvil/packaging/helpers/pip_helper.py @@ -47,6 +47,15 @@ def create_requirement(name, version=None): return pkg_resources.Requirement.parse(name) +def extract(line): + return pip_req.InstallRequirement.from_line(line) + + +def extract_requirement(line): + req = extract(line) + return req.req + + def get_directory_details(path): if not sh.isdir(path): raise IOError("Can not detail non-existent directory %s" % (path)) @@ -57,7 +66,7 @@ def get_directory_details(path): if cache_key in EGGS_DETAILED: return EGGS_DETAILED[cache_key] - req = pip_req.InstallRequirement.from_line(path) + req = extract(path) req.source_dir = path req.run_egg_info() diff --git a/anvil/packaging/yum.py b/anvil/packaging/yum.py index 7e70363f..a6a9e6c7 100644 --- a/anvil/packaging/yum.py +++ b/anvil/packaging/yum.py @@ -176,14 +176,19 @@ class YumDependencyHandler(base.DependencyHandler): def filter_download_requires(self): yum_map = self._get_yum_available() - nopips = [pkg_resources.Requirement.parse(name).key + no_pips = [pkg_resources.Requirement.parse(name).key for name in self.python_names] + pip_origins = {} + for line in self.pips_to_install: + req = pip_helper.extract_requirement(line) + pip_origins[req.key] = line + pips_to_download = [] - req_to_install = [pkg_resources.Requirement.parse(pkg) - for pkg in self.pips_to_install] + req_to_install = [pip_helper.extract_requirement(line) + for line in self.pips_to_install] req_to_install = [req for req in req_to_install - if req.key not in nopips] + if req.key not in no_pips] requested_names = [req.key for req in req_to_install] rpm_to_install = self._convert_names_python2rpm(requested_names) @@ -192,7 +197,7 @@ class YumDependencyHandler(base.DependencyHandler): for (req, rpm_name) in zip(req_to_install, rpm_to_install): (version, repo) = self._find_yum_match(yum_map, req, rpm_name) if not repo: - pips_to_download.append(str(req)) + pips_to_download.append(pip_origins[req.key]) else: satisfied_list.append((req, rpm_name, version, repo)) diff --git a/tools/multipip b/tools/multipip index 251fcb50..81d82dc9 100755 --- a/tools/multipip +++ b/tools/multipip @@ -285,7 +285,7 @@ def join_requirements(options): "pip freeze") incompatible_requirement(frozen_req, joined_req) joined_req = frozen_req - joined_requirements.append(joined_req.req) + joined_requirements.append(joined_req) segment_ok = False lower_version = None @@ -323,7 +323,14 @@ def join_requirements(options): def print_requirements(): - for req in sorted(joined_requirements, key=lambda x: x.key): + formatted_requirements = [] + for req in joined_requirements: + if req.url: + req = "%s#egg=%s" % (req.url, req.req) + else: + req = str(req.req) + formatted_requirements.append(req) + for req in sorted(formatted_requirements): print req