x/anvil
Code Issues Proposed changes

When a requirement url is used don't lose it.

Browse Source 
Previously it seems like we lost requirement
urls and then downloading said requirment url
would fail (since the url is typically provided
if the package isn't on pypi yet). This seems
to be happening for the latests nova requirement
which seems to try to pull in a special oslo
config version.

Change-Id: I30acea47f07f6d189fd63ab9e90434f1eb4e4e2d
This commit is contained in:
Joshua Harlow 2013-06-19 21:53:15 -07:00
parent e4c595a2df
commit 8806ac2a53
4 changed files with 47 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
anvil/packaging/base.py
View File

@ -180,7 +180,7 @@ class DependencyHandler(object):
new_lines = []
for line in old_lines:
try:
req = pkg_resources.Requirement.parse(line)
req = pip_helper.extract_requirement(line)
new_lines.append(str(forced_by_key[req.key]))
except:
# we don't force the package or it has a bad format
@ -213,10 +213,11 @@ class DependencyHandler(object):
LOG.warning(line)
if line.endswith(": incompatible requirements"):
forced_keys.add(line.split(":", 1)[0].lower())
self.pips_to_install = [
pkg
for pkg in utils.splitlines_not_empty(output[0])
if pkg.lower() not in OPENSTACK_PACKAGES]
self.pips_to_install = []
for line in utils.splitlines_not_empty(output[0]):
req = pip_helper.extract_requirement(line)
if req.key not in OPENSTACK_PACKAGES:
self.pips_to_install.append(line)
sh.write_file(self.gathered_requires_filename,
"\n".join(self.pips_to_install))
if not self.pips_to_install:
@ -229,8 +230,8 @@ class DependencyHandler(object):
logger=LOG,
header="Full known python dependency list")
self.forced_packages = []
for pip in self.pips_to_install:
req = pkg_resources.Requirement.parse(pip)
for line in self.pips_to_install:
req = pip_helper.extract_requirement(line)
if req.key in forced_keys:
self.forced_packages.append(req)
sh.write_file(self.forced_requires_filename,
@ -290,11 +291,9 @@ class DependencyHandler(object):
download_requires_filename = sh.joinpths(self.deps_dir,
"download-requires")
raw_pips_to_download = self.filter_download_requires()
pips_to_download = [pkg_resources.Requirement.parse(str(p.strip()))
for p in raw_pips_to_download if p.strip()]
sh.write_file(download_requires_filename,
"\n".join(str(req) for req in pips_to_download))
if not pips_to_download:
"\n".join(str(req) for req in raw_pips_to_download))
if not raw_pips_to_download:
return ([], [])
pip_dir = sh.joinpths(self.deps_dir, "pip")
pip_download_dir = sh.joinpths(pip_dir, "download")
@ -303,19 +302,19 @@ class DependencyHandler(object):
if clear_cache:
sh.deldir(pip_cache_dir)
pip_failures = []
how_many = len(pips_to_download)
for attempt in xrange(self.MAX_PIP_DOWNLOAD_ATTEMPTS):
# NOTE(aababilov): pip has issues with already downloaded files
sh.deldir(pip_download_dir)
sh.mkdir(pip_download_dir, recurse=True)
sh.deldir(pip_build_dir)
header = "Downloading %s python dependencies (attempt %s)"
header = header % (len(raw_pips_to_download), attempt)
utils.log_iterable(sorted(raw_pips_to_download),
logger=LOG,
header=("Downloading %s python dependencies "
"(attempt %s)" % (how_many, attempt)))
header=header)
failed = False
try:
self._try_download_dependencies(attempt, pips_to_download,
self._try_download_dependencies(attempt, raw_pips_to_download,
pip_download_dir,
pip_cache_dir, pip_build_dir)
pip_failures = []
@ -327,8 +326,10 @@ class DependencyHandler(object):
break
if pip_failures:
raise pip_failures[-1]
self._examine_download_dir(pips_to_download, pip_download_dir)
pips_downloaded = [pip_helper.extract_requirement(p)
for p in raw_pips_to_download]
self._examine_download_dir(pips_downloaded, pip_download_dir)
for filename in sh.listdir(pip_download_dir, files_only=True):
sh.move(filename, self.download_dir)
what_downloaded = sh.listdir(self.download_dir, files_only=True)
return (pips_to_download, what_downloaded)
return (pips_downloaded, what_downloaded)

11
anvil/packaging/helpers/pip_helper.py
View File

@ -47,6 +47,15 @@ def create_requirement(name, version=None):
return pkg_resources.Requirement.parse(name)
def extract(line):
return pip_req.InstallRequirement.from_line(line)
def extract_requirement(line):
req = extract(line)
return req.req
def get_directory_details(path):
if not sh.isdir(path):
raise IOError("Can not detail non-existent directory %s" % (path))
@ -57,7 +66,7 @@ def get_directory_details(path):
if cache_key in EGGS_DETAILED:
return EGGS_DETAILED[cache_key]
req = pip_req.InstallRequirement.from_line(path)
req = extract(path)
req.source_dir = path
req.run_egg_info()

15
anvil/packaging/yum.py
View File

@ -176,14 +176,19 @@ class YumDependencyHandler(base.DependencyHandler):
def filter_download_requires(self):
yum_map = self._get_yum_available()
nopips = [pkg_resources.Requirement.parse(name).key
no_pips = [pkg_resources.Requirement.parse(name).key
for name in self.python_names]
pip_origins = {}
for line in self.pips_to_install:
req = pip_helper.extract_requirement(line)
pip_origins[req.key] = line
pips_to_download = []
req_to_install = [pkg_resources.Requirement.parse(pkg)
for pkg in self.pips_to_install]
req_to_install = [pip_helper.extract_requirement(line)
for line in self.pips_to_install]
req_to_install = [req for req in req_to_install
if req.key not in nopips]
if req.key not in no_pips]
requested_names = [req.key for req in req_to_install]
rpm_to_install = self._convert_names_python2rpm(requested_names)
@ -192,7 +197,7 @@ class YumDependencyHandler(base.DependencyHandler):
for (req, rpm_name) in zip(req_to_install, rpm_to_install):
(version, repo) = self._find_yum_match(yum_map, req, rpm_name)
if not repo:
pips_to_download.append(str(req))
pips_to_download.append(pip_origins[req.key])
else:
satisfied_list.append((req, rpm_name, version, repo))

11
tools/multipip
View File

@ -285,7 +285,7 @@ def join_requirements(options):
"pip freeze")
incompatible_requirement(frozen_req, joined_req)
joined_req = frozen_req
joined_requirements.append(joined_req.req)
joined_requirements.append(joined_req)
segment_ok = False
lower_version = None
@ -323,7 +323,14 @@ def join_requirements(options):
def print_requirements():
for req in sorted(joined_requirements, key=lambda x: x.key):
formatted_requirements = []
for req in joined_requirements:
if req.url:
req = "%s#egg=%s" % (req.url, req.req)
else:
req = str(req.req)
formatted_requirements.append(req)
for req in sorted(formatted_requirements):
print req