x/anvil
Code Issues Proposed changes

Use the pip-download helper for pip dependency downloading

Browse Source 
Change-Id: Iaff3749072dd10847b3153bf553d8bae7d39718e
This commit is contained in:
Joshua Harlow
2013-08-04 10:34:48 -07:00
parent 6966143bdd
commit 18ebd229c8
3 changed files with 44 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
anvil/packaging/base.py
View File

@@ -71,23 +71,25 @@ class DependencyHandler(object):
MAX_PIP_DOWNLOAD_ATTEMPTS = 4
multipip_executable = sh.which("multipip", ["tools/"])
pip_executable = sh.which_first(['pip-python', 'pip'])
pipdownload_executable = sh.which("pip-download", ["tools"])
def __init__(self, distro, root_dir, instances, opts=None):
self.distro = distro
self.root_dir = root_dir
self.instances = instances
self.opts = opts or {}
# Various paths we will use while operating
self.deps_dir = sh.joinpths(self.root_dir, "deps")
self.downloaded_flag_file = sh.joinpths(self.deps_dir, "pip-downloaded")
self.download_dir = sh.joinpths(self.deps_dir, "download")
self.log_dir = sh.joinpths(self.deps_dir, "output")
self.gathered_requires_filename = sh.joinpths(
self.deps_dir, "pip-requires")
self.forced_requires_filename = sh.joinpths(
self.deps_dir, "forced-requires")
# list of requirement strings
self.gathered_requires_filename = sh.joinpths(self.deps_dir, "pip-requires")
self.forced_requires_filename = sh.joinpths(self.deps_dir, "forced-requires")
self.download_requires_filename = sh.joinpths(self.deps_dir, "download-requires")
# List of requirement strings
self.pips_to_install = []
self.forced_packages = []
# Instances to there app directory (with a setup.py inside)
self.package_dirs = self._get_package_dirs(instances)
# Instantiate this as late as we can.
self._python_names = None
@@ -249,19 +251,13 @@ class DependencyHandler(object):
"""
return self.pips_to_install
def _try_download_dependencies(self, attempt, pips_to_download,
pip_download_dir,
pip_cache_dir,
pip_build_dir):
pips_to_download = [str(p) for p in pips_to_download]
def _try_download_dependencies(self, attempt, pips_to_download, pip_download_dir):
cmdline = [
self.pip_executable,
"install",
"--download", pip_download_dir,
"--download-cache", pip_cache_dir,
"--build", pip_build_dir,
self.pipdownload_executable,
'-d', pip_download_dir,
'-v',
]
cmdline.extend(sorted(pips_to_download))
cmdline.extend(sorted([str(p) for p in pips_to_download]))
out_filename = sh.joinpths(self.log_dir,
"pip-download-attempt-%s.log" % (attempt))
sh.execute_save_output(cmdline, out_filename=out_filename)
@@ -302,38 +298,29 @@ class DependencyHandler(object):
"""
# NOTE(aababilov): do not drop download_dir - it can be reused
sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter)
download_requires_filename = sh.joinpths(self.deps_dir, "download-requires")
raw_pips_to_download = self.filter_download_requires()
sh.write_file(download_requires_filename,
"\n".join(str(req) for req in raw_pips_to_download))
if not raw_pips_to_download:
pips_to_download = self.filter_download_requires()
sh.write_file(self.download_requires_filename,
"\n".join([str(req) for req in pips_to_download]))
if not pips_to_download:
return ([], [])
# NOTE(aababilov): user could have changed persona, so,
# check that all requirements are downloaded
if (sh.isfile(self.downloaded_flag_file) and
self._requirements_satisfied(raw_pips_to_download, self.download_dir)):
self._requirements_satisfied(pips_to_download, self.download_dir)):
LOG.info("All python dependencies have been already downloaded")
else:
pip_dir = sh.joinpths(self.deps_dir, "pip")
pip_download_dir = sh.joinpths(pip_dir, "download")
pip_build_dir = sh.joinpths(pip_dir, "build")
# NOTE(aababilov): do not clean the cache, it is always useful
pip_cache_dir = sh.joinpths(self.deps_dir, "pip-cache")
pip_failures = []
for attempt in xrange(self.MAX_PIP_DOWNLOAD_ATTEMPTS):
# NOTE(aababilov): pip has issues with already downloaded files
sh.deldir(pip_dir)
sh.mkdirslist(pip_download_dir, tracewriter=self.tracewriter)
for filename in sh.listdir(self.download_dir, files_only=True):
sh.unlink(filename)
header = "Downloading %s python dependencies (attempt %s)"
header = header % (len(raw_pips_to_download), attempt)
utils.log_iterable(sorted(raw_pips_to_download),
logger=LOG,
header=header)
header = header % (len(pips_to_download), attempt + 1)
utils.log_iterable(sorted(pips_to_download), logger=LOG, header=header)
failed = False
try:
self._try_download_dependencies(attempt, raw_pips_to_download,
pip_download_dir,
pip_cache_dir, pip_build_dir)
self._try_download_dependencies(attempt + 1, pips_to_download,
self.download_dir)
pip_failures = []
except exc.ProcessExecutionError as e:
LOG.exception("Failed downloading python dependencies")
@@ -341,15 +328,11 @@ class DependencyHandler(object):
failed = True
if not failed:
break
for filename in sh.listdir(pip_download_dir, files_only=True):
sh.move(filename, self.download_dir, force=True)
sh.deldir(pip_dir)
if pip_failures:
raise pip_failures[-1]
# NOTE(harlowja): Mark that we completed downloading successfully
sh.touch_file(self.downloaded_flag_file, die_if_there=False,
quiet=True, tracewriter=self.tracewriter)
pips_downloaded = [pip_helper.extract_requirement(p)
for p in raw_pips_to_download]
pips_downloaded = [pip_helper.extract_requirement(p) for p in pips_to_download]
self._examine_download_dir(pips_downloaded, self.download_dir)
what_downloaded = sh.listdir(self.download_dir, files_only=True)
return (pips_downloaded, what_downloaded)
return (pips_downloaded, sh.listdir(self.download_dir, files_only=True))

9
anvil/packaging/yum.py
View File

@@ -75,10 +75,6 @@ class YumDependencyHandler(base.DependencyHandler):
}
REPO_FN = "anvil.repo"
YUM_REPO_DIR = "/etc/yum.repos.d/"
BANNED_PACKAGES = [
'distribute',
'setuptools',
]
SRC_REPOS = {
'anvil': 'anvil-source',
"anvil-deps": "anvil-deps-source",
@@ -100,13 +96,15 @@ class YumDependencyHandler(base.DependencyHandler):
def __init__(self, distro, root_dir, instances, opts=None):
super(YumDependencyHandler, self).__init__(distro, root_dir, instances, opts)
# Various paths we will use while operating
self.rpmbuild_dir = sh.joinpths(self.deps_dir, "rpmbuild")
self.deps_repo_dir = sh.joinpths(self.deps_dir, "openstack-deps")
self.deps_src_repo_dir = sh.joinpths(self.deps_dir, "openstack-deps-sources")
self.anvil_repo_filename = sh.joinpths(self.deps_dir, self.REPO_FN)
self.helper = yum_helper.Helper(self.log_dir)
self.rpm_sources_dir = sh.joinpths(self.rpmbuild_dir, "SOURCES")
self.anvil_repo_dir = sh.joinpths(self.root_dir, "repo")
# We inspect yum for packages, this helper allows us to do this.
self.helper = yum_helper.Helper(self.log_dir)
self._no_remove = None
def py2rpm_start_cmdline(self):
@@ -330,7 +328,6 @@ class YumDependencyHandler(base.DependencyHandler):
# build or can satisfy by other means
no_pips = [pkg_resources.Requirement.parse(name).key
for name in self.python_names]
no_pips.extend(self.BANNED_PACKAGES)
yum_map = self._get_yum_available()
pips_keys = set([p.key for p in pips_downloaded])