x/anvil
Code Issues Proposed changes

Merge "Show and filter extra pip dependencies downloaded."

Browse Source
This commit is contained in:
Jenkins 2013-06-11 01:11:56 +00:00 committed by Gerrit Code Review
commit 8d8da96137
4 changed files with 119 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
anvil/packaging/base.py
View File

@ -23,6 +23,7 @@ import pkg_resources
from anvil import colorizer
from anvil import exceptions as exc
from anvil import log as logging
from anvil.packaging.helpers import pip_helper
from anvil import shell as sh
from anvil import utils
@ -236,6 +237,17 @@ class DependencyHandler(object):
out_filename = sh.joinpths(self.log_dir, download_filename)
sh.execute_save_output(cmdline, out_filename=out_filename)
def _examine_download_dir(self, pips_to_download, pip_download_dir):
pip_names = set([p.key for p in pips_to_download])
what_downloaded = sh.listdir(pip_download_dir, files_only=True)
LOG.info("Validating %s files that were downloaded.",
len(what_downloaded))
for filename in what_downloaded:
pkg_details = pip_helper.get_archive_details(filename)
req = pkg_details['req']
if req.key not in pip_names:
LOG.info("Dependency %s was automatically included.", req)
def download_dependencies(self, clear_cache=False):
"""Download dependencies from `$deps_dir/download-requires`.
@ -252,7 +264,7 @@ class DependencyHandler(object):
sh.write_file(download_requires_filename,
"\n".join(str(req) for req in pips_to_download))
if not pips_to_download:
return []
return ([], [])
pip_dir = sh.joinpths(self.deps_dir, "pip")
pip_download_dir = sh.joinpths(pip_dir, "download")
pip_build_dir = sh.joinpths(pip_dir, "build")
@ -284,6 +296,8 @@ class DependencyHandler(object):
break
if pip_failures:
raise pip_failures[-1]
self._examine_download_dir(pips_to_download, pip_download_dir)
for filename in sh.listdir(pip_download_dir, files_only=True):
sh.move(filename, self.download_dir)
return sh.listdir(self.download_dir, files_only=True)
what_downloaded = sh.listdir(self.download_dir, files_only=True)
return (pips_to_download, what_downloaded)

39
anvil/packaging/helpers/pip_helper.py
View File

@ -15,14 +15,20 @@
# under the License.
import copy
import os
import pkg_resources
from pip import util as pip_util
from pip import req as pip_req
from anvil import log as logging
from anvil import shell as sh
from anvil import utils
LOG = logging.getLogger(__name__)
FREEZE_CMD = ['freeze', '--local']
FILES_DETAILED = {}
def create_requirement(name, version=None):
@ -42,6 +48,39 @@ def create_requirement(name, version=None):
return pkg_resources.Requirement.parse(name)
def get_archive_details(filename):
if not os.path.isfile(filename):
raise IOError("Can not detail non-existent file %s" % (filename))
# Check if we already got the details of this file previously
#
# A filename and size cache key should be good enough to match against
# most non-malicous files...
cache_key = "%s:%s" % (sh.basename(filename), sh.getsize(filename))
if cache_key in FILES_DETAILED:
return FILES_DETAILED[cache_key]
# Get pip to get us the egg-info.
with utils.tempdir() as td:
filename = sh.copy(filename, sh.joinpths(td, sh.basename(filename)))
extract_to = sh.mkdir(sh.joinpths(td, 'build'))
pip_util.unpack_file(filename, extract_to, content_type='', link='')
req = pip_req.InstallRequirement.from_line(extract_to)
req.source_dir = extract_to
req.run_egg_info()
# Selectively extract pieces of the install requirement
details = {
'req': req.req,
'dependencies': req.requirements,
'name': req.name,
'pkg_info': req.pkg_info,
'dependency_links': req.dependency_links,
}
FILES_DETAILED[cache_key] = details
return details
def _skip_requirement(line):
# Skip blank lines or comment lines
if not len(line):

82
anvil/packaging/yum.py
View File

@ -23,6 +23,7 @@ from datetime import datetime
from anvil import colorizer
from anvil import log as logging
from anvil.packaging import base
from anvil.packaging.helpers import pip_helper
from anvil.packaging.helpers import yum_helper
from anvil import shell as sh
from anvil import trace as tr
@ -98,13 +99,24 @@ class YumDependencyHandler(base.DependencyHandler):
self._build_openstack()
self._create_deps_repo()
def filter_download_requires(self):
def _get_yum_available(self):
yum_map = {}
for pkg in yum_helper.Helper().get_available():
for pkg in self.helper.get_available():
for provides in pkg.provides:
yum_map.setdefault(provides[0], set()).add(
(pkg.version, pkg.repo))
pkg_info = (pkg.version, pkg.repo)
yum_map.setdefault(provides[0], set()).add(pkg_info)
return yum_map
@staticmethod
def _find_yum_match(yum_map, req, rpm_name):
yum_versions = yum_map.get(rpm_name, [])
for (version, repo) in yum_versions:
if version in req:
return (version, repo)
return (None, None)
def filter_download_requires(self):
yum_map = self._get_yum_available()
nopips = [pkg_resources.Requirement.parse(name).key
for name in self.python_names]
@ -119,15 +131,11 @@ class YumDependencyHandler(base.DependencyHandler):
satisfied_list = []
for (req, rpm_name) in zip(req_to_install, rpm_to_install):
yum_versions = yum_map.get(rpm_name, [])
satisfied = False
for (version, repo) in yum_versions:
if version in req:
satisfied = True
satisfied_list.append((req, rpm_name, version, repo))
break
if not satisfied:
(version, repo) = self._find_yum_match(yum_map, req, rpm_name)
if not repo:
pips_to_download.append(str(req))
else:
satisfied_list.append((req, rpm_name, version, repo))
if satisfied_list:
# Organize by repo
@ -249,19 +257,49 @@ BuildArch: noarch
sh.execute(cmdline)
def _build_dependencies(self):
package_files = self.download_dependencies()
(pips_downloaded, package_files) = self.download_dependencies()
def filter_files(package_files):
for p in package_files:
banned = False
for k in self.BANNED_PACKAGES:
if k in p.lower():
banned = True
if banned:
# Analyze what was downloaded and eject things that were downloaded
# by pip as a dependency of a download but which we do not want to
# build or can satisfy by other means
no_pips = [pkg_resources.Requirement.parse(name).key
for name in self.python_names]
no_pips.extend(self.BANNED_PACKAGES)
yum_map = self._get_yum_available()
pips_keys = set([p.key for p in pips_downloaded])
def filter_package_files(package_files):
package_reqs = []
package_keys = []
for filename in package_files:
package_details = pip_helper.get_archive_details(filename)
package_reqs.append(package_details['req'])
package_keys.append(package_details['req'].key)
package_rpm_names = self._convert_names_python2rpm(package_keys)
filtered_files = []
for (filename, req, rpm_name) in zip(package_files, package_reqs,
package_rpm_names):
if req.key in no_pips:
LOG.info(("Dependency %s was downloaded additionally "
"but it is disallowed."), req)
continue
yield p
if req.key in pips_keys:
filtered_files.append(filename)
continue
# See if pip tried to download it but we already can satisfy
# it via yum and avoid building it in the first place...
(version, repo) = self._find_yum_match(yum_map, req, rpm_name)
if not repo:
filtered_files.append(filename)
else:
LOG.info(("Dependency %s was downloaded additionally "
"but it can be satisfied by %s from repository "
"%s instead."), req, colorizer.quote(rpm_name),
colorizer.quote(repo))
return filtered_files
package_files = [f for f in filter_files(package_files)]
LOG.info("Filtering %s downloaded files.", len(package_files))
package_files = filter_package_files(package_files)
if not package_files:
LOG.info("No RPM packages of OpenStack dependencies to build")
return

4
anvil/shell.py
View File

@ -47,6 +47,10 @@ SUDO_GID = env.get_key('SUDO_GID')
# Set only once
IS_DRYRUN = None
# Take over some functions directly from os.path
getsize = os.path.getsize
class Process(psutil.Process):
def __str__(self):