From 0e9eff8c33ec68121fd2b797222c6d49390da506 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 16 Nov 2012 10:49:42 -0800 Subject: [PATCH] Grizzly cleanups and requirements checking enhancements. --- anvil/components/__init__.py | 198 +++++++++++++++----------- anvil/components/nova.py | 3 - anvil/packager.py | 8 -- anvil/packaging/helpers/pip_helper.py | 105 +++++++------- anvil/packaging/helpers/yum_helper.py | 95 +++++++----- anvil/packaging/pip.py | 77 +++++----- anvil/packaging/rpm.py | 6 +- anvil/packaging/yum.py | 73 +++++++--- anvil/shell.py | 2 - conf/distros/rhel.yaml | 16 ++- conf/personas/in-a-box/basic-web.yaml | 9 +- conf/personas/in-a-box/basic.yaml | 3 +- 12 files changed, 338 insertions(+), 257 deletions(-) diff --git a/anvil/components/__init__.py b/anvil/components/__init__.py index e3b9af2d..54d7c154 100644 --- a/anvil/components/__init__.py +++ b/anvil/components/__init__.py @@ -31,11 +31,10 @@ # under the License. import functools +import os import re import weakref -from pkg_resources import Requirement - from anvil import cfg from anvil import colorizer from anvil import component @@ -52,6 +51,8 @@ from anvil import utils from anvil.packaging import pip +from anvil.packaging.helpers import pip_helper + LOG = logging.getLogger(__name__) #### @@ -73,10 +74,22 @@ class ProgramStatus(object): #### Utils... #### +# Cache of accessed packagers +_PACKAGERS = {} + def make_packager(package, default_class, **kwargs): - cls = packager.get_packager_class(package, default_class) - return cls(**kwargs) + packager_name = package.get('packager_name') or '' + packager_name = packager_name.strip() + if packager_name: + packager_cls = importer.import_entry_point(packager_name) + else: + packager_cls = default_class + if packager_cls in _PACKAGERS: + return _PACKAGERS[packager_cls] + p = packager_cls(**kwargs) + _PACKAGERS[packager_cls] = p + return p #### @@ -93,15 +106,6 @@ class PkgInstallComponent(component.Component): def _get_download_config(self): return None - def _clear_package_duplicates(self, pkg_list): - dup_free_list = [] - names_there = set() - for pkg in pkg_list: - if pkg['name'] not in names_there: - dup_free_list.append(pkg) - names_there.add(pkg['name']) - return dup_free_list - def _get_download_location(self): key = self._get_download_config() if not key: @@ -157,7 +161,6 @@ class PkgInstallComponent(component.Component): if 'packages' in values: LOG.debug("Extending package list with packages for subsystem: %r", name) pkg_list.extend(values.get('packages')) - pkg_list = self._clear_package_duplicates(pkg_list) return pkg_list def install(self): @@ -169,7 +172,8 @@ class PkgInstallComponent(component.Component): header="Setting up %s distribution packages" % (len(pkg_names))) with utils.progress_bar('Installing', len(pkgs)) as p_bar: for (i, p) in enumerate(pkgs): - installer = make_packager(p, self.distro.package_manager_class, distro=self.distro) + installer = make_packager(p, self.distro.package_manager_class, + distro=self.distro) self.tracewriter.package_installed(p) installer.install(p) p_bar.update(i + 1) @@ -177,13 +181,15 @@ class PkgInstallComponent(component.Component): def pre_install(self): pkgs = self.packages for p in pkgs: - installer = make_packager(p, self.distro.package_manager_class, distro=self.distro) + installer = make_packager(p, self.distro.package_manager_class, + distro=self.distro) installer.pre_install(p, self.params) def post_install(self): pkgs = self.packages for p in pkgs: - installer = make_packager(p, self.distro.package_manager_class, distro=self.distro) + installer = make_packager(p, self.distro.package_manager_class, + distro=self.distro) installer.post_install(p, self.params) @property @@ -298,64 +304,73 @@ class PythonInstallComponent(PkgInstallComponent): pip_pkg_list = [] return pip_pkg_list - def _match_pip_requires(self, pip_name): + def _match_pip_requires(self, pip_req): - # TODO(harlowja) Is this a bug?? that this is needed? - def pip_match(in1, in2): - in1 = in1.replace("-", "_") - in1 = in1.lower() - in2 = in2.replace('-', '_') - in2 = in2.lower() - return in1 == in2 + def pip_use(who, there_pip): + if there_pip.key != pip_req.key: + return False + if not len(pip_req.specs): + # No version/restrictions specified + return True + there_version = None + if there_pip.version is not None: + there_version = str(there_pip.version) + if there_version in pip_req: + return True + # Different possibly incompat. versions found... + if there_version is None: + # Assume pip will install the correct version anyway + if who != self.name: + msg = ("Component %r asked for package '%s'" + " and '%s' is being selected from %r instead...") + LOG.debug(msg, self.name, pip_req, there_pip, who) + return True + else: + if who != self.name: + msg = ("Component %r provides package '%s'" + " but '%s' is being asked for by %r instead...") + LOG.warn(msg, who, there_pip, pip_req, self.name) + return False - pip_found = False - pkg_found = None + LOG.debug("Attempting to find who satisfies pip requirement '%s'", pip_req) # Try to find it in anyones pip -> pkg list - pip2_pkg_mp = { + all_pip_2_pkgs = { self.name: self.pips_to_packages, } + # Gather them all (but only if they activate before me) + # since if they activate after, we can't depend on it + # to satisfy our requirement... for (name, c) in self.instances.items(): if c is self or not c.activated: continue if isinstance(c, (PythonInstallComponent)): - pip2_pkg_mp[name] = c.pips_to_packages - for (who, pips_2_pkgs) in pip2_pkg_mp.items(): + all_pip_2_pkgs[name] = c.pips_to_packages + for (who, pips_2_pkgs) in all_pip_2_pkgs.items(): for pip_info in pips_2_pkgs: - if pip_match(pip_name, pip_info['name']): - pip_found = True - pkg_found = pip_info.get('package') - LOG.debug("Matched pip->pkg %r from component %r", pip_name, who) - break - if pip_found: - break - if pip_found: - return (pkg_found, False) + there_pip = pip.extract_requirement(pip_info) + if not pip_use(who, there_pip): + continue + LOG.debug("Matched pip->pkg '%s' from component %r", there_pip, who) + return (dict(pip_info.get('package')), False) # Ok nobody had it in a pip->pkg mapping # but see if they had it in there pip collection - pip_mp = { - self.name: self._base_pips(), + all_pips = { + self.name: self._base_pips(), # Use base pips to avoid recursion... } for (name, c) in self.instances.items(): if not c.activated or c is self: continue if isinstance(c, (PythonInstallComponent)): - pip_mp[name] = c._base_pips() # pylint: disable=W0212 - pip_found = False - pip_who = None - for (who, pips) in pip_mp.items(): - for pip_info in pips: - if pip_match(pip_info['name'], pip_name): - pip_found = True - pip_who = pip_info - LOG.debug("Matched pip %r from other component %r", pip_name, who) - break - if pip_found: - break - if pip_found: - return (pip_who, True) - + all_pips[name] = c._base_pips() # pylint: disable=W0212 + for (who, there_pips) in all_pips.items(): + for pip_info in there_pips: + there_pip = pip.extract_requirement(pip_info) + if not pip_use(who, there_pip): + continue + LOG.debug("Matched pip '%s' from component %r", there_pip, who) + return (dict(pip_info), True) return (None, False) def _get_mapped_packages(self): @@ -363,7 +378,9 @@ class PythonInstallComponent(PkgInstallComponent): all_pips = [] for fn in self.requires_files: all_pips.extend(self._extract_pip_requires(fn)) - for (_requirement, (pkg_info, from_pip)) in all_pips: + for details in all_pips: + pkg_info = details['package'] + from_pip = details['from_pip'] if from_pip or not pkg_info: continue add_on_pkgs.append(pkg_info) @@ -374,7 +391,9 @@ class PythonInstallComponent(PkgInstallComponent): all_pips = [] for fn in self.requires_files: all_pips.extend(self._extract_pip_requires(fn)) - for (_requirement, (pkg_info, from_pip)) in all_pips: + for details in all_pips: + pkg_info = details['package'] + from_pip = details['from_pip'] if not from_pip or not pkg_info: continue add_on_pips.append(pkg_info) @@ -388,14 +407,12 @@ class PythonInstallComponent(PkgInstallComponent): if 'pips' in values: LOG.debug("Extending pip list with pips for subsystem: %r" % (name)) pip_list.extend(values.get('pips')) - pip_list = self._clear_package_duplicates(pip_list) return pip_list @property def pips(self): pip_list = self._base_pips() pip_list.extend(self._get_mapped_pips()) - pip_list = self._clear_package_duplicates(pip_list) return pip_list def _install_pips(self): @@ -407,7 +424,8 @@ class PythonInstallComponent(PkgInstallComponent): with utils.progress_bar('Installing', len(pips)) as p_bar: for (i, p) in enumerate(pips): self.tracewriter.pip_installed(p) - installer = make_packager(p, pip.Packager, distro=self.distro) + installer = make_packager(p, pip.Packager, + distro=self.distro) installer.install(p) p_bar.update(i + 1) @@ -445,13 +463,15 @@ class PythonInstallComponent(PkgInstallComponent): self._verify_pip_requires() PkgInstallComponent.pre_install(self) for p in self.pips: - installer = make_packager(p, pip.Packager, distro=self.distro) + installer = make_packager(p, pip.Packager, + distro=self.distro) installer.pre_install(p, self.params) def post_install(self): PkgInstallComponent.post_install(self) for p in self.pips: - installer = make_packager(p, pip.Packager, distro=self.distro) + installer = make_packager(p, pip.Packager, + distro=self.distro) installer.post_install(p, self.params) def _install_python_setups(self): @@ -483,28 +503,30 @@ class PythonInstallComponent(PkgInstallComponent): if not sh.isfile(fn): return [] LOG.debug("Resolving dependencies from %s.", colorizer.quote(fn)) - pips_needed = [] - for line in sh.load_file(fn).splitlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - pips_needed.append(Requirement.parse(line)) - if not pips_needed: - return [] + pips_needed = pip_helper.parse_requirements(sh.load_file(fn)) matchings = [] - for requirement in pips_needed: - matchings.append([requirement, self._match_pip_requires(requirement.project_name)]) + for req in pips_needed: + (pkg_info, from_pip) = self._match_pip_requires(req) + matchings.append({ + 'requirement': req, + 'package': pkg_info, + 'from_pip': from_pip, + 'needed_by': fn, + }) return matchings def _verify_pip_requires(self): all_pips = [] for fn in self.requires_files: all_pips.extend(self._extract_pip_requires(fn)) - for (requirement, (pkg_info, _from_pip)) in all_pips: + for details in all_pips: + req = details['requirement'] + needed_by = details['needed_by'] + pkg_info = details['package'] if not pkg_info: - raise excp.DependencyException(("Pip dependency '%s' is not translatable to a listed" + raise excp.DependencyException(("Pip dependency '%s' needed by '%s' is not translatable to a listed" " (from this or previously activated components) pip package" - ' or a pip->package mapping!') % (requirement)) + ' or a pip->package mapping!') % (req, needed_by)) def install(self): PkgInstallComponent.install(self) @@ -723,7 +745,8 @@ class PkgUninstallComponent(component.Component): with utils.progress_bar('Uninstalling', len(pkgs), reverse=True) as p_bar: for (i, p) in enumerate(pkgs): uninstaller = make_packager(p, self.distro.package_manager_class, - distro=self.distro, remove_default=self.purge_packages) + distro=self.distro, + remove_default=self.purge_packages) if uninstaller.remove(p): which_removed.append(p['name']) p_bar.update(i + 1) @@ -766,7 +789,8 @@ class PythonUninstallComponent(PkgUninstallComponent): for (i, p) in enumerate(pips): try: uninstaller = make_packager(p, pip.Packager, - distro=self.distro, remove_default=self.purge_packages) + distro=self.distro, + remove_default=self.purge_packages) if uninstaller.remove(p): which_removed.append(p['name']) except excp.ProcessExecutionError as e: @@ -830,8 +854,7 @@ class PythonTestingComponent(component.Component): def _get_env(self): env_addons = {} - app_dir = self.get_option('app_dir') - tox_fn = sh.joinpths(app_dir, 'tox.ini') + tox_fn = sh.joinpths(self.get_option('app_dir'), 'tox.ini') if sh.isfile(tox_fn): # Suck out some settings from the tox file try: @@ -849,14 +872,11 @@ class PythonTestingComponent(component.Component): value = value.strip() if name.lower() != 'virtual_env': env_addons[name] = value + if env_addons: + LOG.debug("From %s we read in %s environment settings:", tox_fn, len(env_addons)) + utils.log_object(env_addons, logger=LOG, level=logging.DEBUG) except IOError: pass - env_addons['NOSE_WITH_OPENSTACK'] = 1 - env_addons['NOSE_OPENSTACK_COLOR'] = 1 - env_addons['NOSE_OPENSTACK_RED'] = 0.05 - env_addons['NOSE_OPENSTACK_YELLOW'] = 0.025 - env_addons['NOSE_OPENSTACK_SHOW_ELAPSED'] = 1 - env_addons['NOSE_OPENSTACK_STDOUT'] = 1 return env_addons def run_tests(self): @@ -867,7 +887,11 @@ class PythonTestingComponent(component.Component): return cmd = self._get_test_command() env = self._get_env() - sh.execute(*cmd, stdout_fh=None, stderr_fh=None, cwd=app_dir, env_overrides=env) + with open(os.devnull, 'w') as null_fh: + if self.get_bool_option("verbose"): + null_fh = None + sh.execute(*cmd, stdout_fh=None, stderr_fh=null_fh, + cwd=app_dir, env_overrides=env) #### diff --git a/anvil/components/nova.py b/anvil/components/nova.py index 9dff7c19..c0a9329b 100644 --- a/anvil/components/nova.py +++ b/anvil/components/nova.py @@ -347,9 +347,6 @@ class NovaTester(comp.PythonTestingComponent): 'test_quantumv2', ] - def _get_env(self): - return {} - def _get_test_command(self): base_cmd = comp.PythonTestingComponent._get_test_command(self) # This doesn't exist in the nosetests (v1.1) in rhel6 diff --git a/anvil/packager.py b/anvil/packager.py index 87ed0bc4..0498971d 100644 --- a/anvil/packager.py +++ b/anvil/packager.py @@ -72,11 +72,3 @@ class Packager(object): @abc.abstractmethod def _install(self, pkg): raise NotImplementedError() - - -def get_packager_class(package_info, default_packager_class=None): - packager_name = package_info.get('packager_name') or '' - packager_name = packager_name.strip() - if not packager_name: - return default_packager_class - return importer.import_entry_point(packager_name) diff --git a/anvil/packaging/helpers/pip_helper.py b/anvil/packaging/helpers/pip_helper.py index 694256eb..d29d6147 100644 --- a/anvil/packaging/helpers/pip_helper.py +++ b/anvil/packaging/helpers/pip_helper.py @@ -15,7 +15,8 @@ # under the License. from distutils import version as vr -from pkg_resources import Requirement + +import pkg_resources from anvil import log as logging from anvil import shell as sh @@ -25,78 +26,68 @@ LOG = logging.getLogger(__name__) FREEZE_CMD = ['freeze', '--local'] -# Cache of whats installed - 'uncached' as needed -_installed_cache = None - - -class LooseRequirement(object): +class Requirement(object): def __init__(self, name, version=None): - self.name = name + self.name = str(name) if version is not None: - self.version = vr.LooseVersion(version) + self.version = vr.LooseVersion(str(version)) else: self.version = None + self.key = self.name.lower() def __str__(self): + name = str(self.name) if self.version is not None: - return "%s (%s)" % (self.name, self.version) - else: - return str(self.name) - - def __contains__(self, version): - if self.version is None: - return True - else: - return version <= self.version + name += "==" + str(self.version) + return name -def uncache(): - global _installed_cache - _installed_cache = None - - -def _list_installed(pip_how): - cmd = [str(pip_how)] + FREEZE_CMD - (stdout, _stderr) = sh.execute(*cmd) - installed = [] - for line in stdout.splitlines(): +def parse_requirements(contents, adjust=False): + lines = [] + for line in contents.splitlines(): line = line.strip() - if not line or line.startswith('#'): + if not len(line) or line.startswith('#'): continue # Don't take editables either... - if line.startswith('-e'): + if line.lower().startswith('-e'): continue - v = None - try: - line_requirements = Requirement.parse(line) - (_cmp, v) = line_requirements.specs[0] - except (ValueError, TypeError) as e: - LOG.warn("Unparseable pip freeze line %s: %s" % (line, e)) - continue - installed.append(LooseRequirement(line_requirements.key, v)) - return installed + lines.append(line) + requires = [] + for req in pkg_resources.parse_requirements(lines): + requires.append(req) + return requires -def whats_installed(pip_how): - global _installed_cache - if _installed_cache is None: - _installed_cache = _list_installed(pip_how) - return _installed_cache +class Helper(object): + # Cache of whats installed list + _installed_cache = None + def __init__(self, pip_how): + self._pip_how = pip_how -def is_installed(pip_how, name, version=None): - if get_installed(pip_how, name, version): - return True - return False + def _list_installed(self): + cmd = [str(self._pip_how)] + FREEZE_CMD + (stdout, _stderr) = sh.execute(*cmd) + return parse_requirements(stdout, True) + @staticmethod + def uncache(): + Helper._installed_cache = None -def get_installed(pip_how, name, version=None): - whats_there = whats_installed(pip_how) - for req in whats_there: - if not (name.lower() == req.name): - continue - if not version: - return req - if version in req: - return req - return None + def whats_installed(self): + if Helper._installed_cache is None: + Helper._installed_cache = self._list_installed() + return list(Helper._installed_cache) + + def is_installed(self, name): + if self.get_installed(name): + return True + return False + + def get_installed(self, name): + whats_there = self.whats_installed() + for whats_installed in whats_there: + if not (name.lower() == whats_installed.key): + continue + return whats_installed + return None diff --git a/anvil/packaging/helpers/yum_helper.py b/anvil/packaging/helpers/yum_helper.py index 9bc03f13..4168ece5 100644 --- a/anvil/packaging/helpers/yum_helper.py +++ b/anvil/packaging/helpers/yum_helper.py @@ -18,47 +18,66 @@ from anvil import shell as sh # See http://yum.baseurl.org/api/yum-3.2.26/yum-module.html from yum import YumBase + from yum.packages import PackageObject -# Cache of yumbase object -_yum_base = None + +class Requirement(object): + def __init__(self, name, version): + self.name = str(name) + self.version = version + + def __str__(self): + name = self.name + if self.version is not None: + name += "-%s" % (self.version) + return name + + @property + def package(self): + # Form a 'fake' rpm package that + # can be used to compare against + # other rpm packages using the + # standard rpm routines + my_pkg = PackageObject() + my_pkg.name = self.name + if self.version is not None: + my_pkg.version = str(self.version) + return my_pkg -def _make_yum_base(): - global _yum_base - if _yum_base is None: - # This seems needed... +class Helper(object): + # Cache of yumbase object + _yum_base = None + + @staticmethod + def _get_yum_base(): + if Helper._yum_base is None: + # This 'root' seems needed... + # otherwise 'cannot open Packages database in /var/lib/rpm' starts to happen + with sh.Rooted(True): + _yum_base = YumBase() + _yum_base.setCacheDir(force=True) + Helper._yum_base = _yum_base + return Helper._yum_base + + def is_installed(self, name): + if len(self.get_installed(name)): + return True + else: + return False + + def get_installed(self, name): + base = Helper._get_yum_base() + # This 'root' seems needed... # otherwise 'cannot open Packages database in /var/lib/rpm' starts to happen + # even though we are just doing a read-only operation, which + # is pretty odd... with sh.Rooted(True): - _yum_base = YumBase() - _yum_base.setCacheDir(force=True) - return _yum_base - - -def is_installed(name, version=None): - if get_installed(name, version): - return True - else: - return False - - -def get_installed(name, version=None): - # This seems needed... - # otherwise 'cannot open Packages database in /var/lib/rpm' starts to happen - with sh.Rooted(True): - yb = _make_yum_base() - pkg_obj = yb.doPackageLists(pkgnarrow='installed', - ignore_case=True, patterns=[name]) - whats_installed = pkg_obj.installed - if not whats_installed: - return None - # Compare whats installed to a fake package that will - # represent what might be installed... - fake_pkg = PackageObject() - fake_pkg.name = name - if version: - fake_pkg.version = str(version) - for installed_pkg in whats_installed: - if installed_pkg.verGE(fake_pkg): - return installed_pkg - return None + pkgs = base.doPackageLists(pkgnarrow='installed', + ignore_case=True, patterns=[name]) + if pkgs.installed: + whats_installed = list(pkgs.installed) + else: + whats_installed = [] + return whats_installed diff --git a/anvil/packaging/pip.py b/anvil/packaging/pip.py index 6287db00..56ebed5e 100644 --- a/anvil/packaging/pip.py +++ b/anvil/packaging/pip.py @@ -14,50 +14,61 @@ # License for the specific language governing permissions and limitations # under the License. +from anvil import exceptions as excp from anvil import log as logging from anvil import packager as pack from anvil import shell as sh from anvil.packaging.helpers import pip_helper +import pkg_resources + LOG = logging.getLogger(__name__) PIP_UNINSTALL_CMD_OPTS = ['-y', '-q'] PIP_INSTALL_CMD_OPTS = ['-q'] -NAMED_VERSION_TEMPL = "%s == %s" + + +def extract_requirement(pkg_info): + p_name = pkg_info.get('name', '') + p_name = p_name.strip() + p_name = pkg_resources.safe_name(p_name) + if not p_name: + raise ValueError("Pip requirement provided with an empty name") + p_version = pkg_info.get('version') + if p_version is not None: + if isinstance(p_version, (int, float, long)): + p_version = str(p_version) + if isinstance(p_version, (str, basestring)): + p_version = pkg_resources.safe_version(p_version) + else: + raise TypeError("Pip requirement version must be a string or numeric type") + return pip_helper.Requirement(p_name, p_version) class Packager(pack.Packager): - - def _make_pip_name(self, name, version): - if not version: - return str(name) - else: - return NAMED_VERSION_TEMPL % (name, version) + def __init__(self, distro, remove_default=False): + pack.Packager.__init__(self, distro, remove_default) + self.helper = pip_helper.Helper(self._get_pip_command()) def _get_pip_command(self): return self.distro.get_command_config('pip') - def _anything_there(self, pkg): - pkg_there = pip_helper.get_installed(self._get_pip_command(), - pkg['name'], pkg.get('version')) - if not pkg_there: + def _anything_there(self, pip): + wanted_pip = extract_requirement(pip) + pip_there = self.helper.get_installed(wanted_pip.name) + if not pip_there: + # Nothing installed return None - if 'options' in pkg and 'version' in pkg: - # Ensure exact version if options - wanted_pkg = pip_helper.LooseRequirement(pkg['name'], - pkg.get('version')) - wanted_ver = wanted_pkg.version - if wanted_ver == pkg_there.version and wanted_ver is not None: - return pkg_there - else: - # Mismatched version - return None - elif 'options' in pkg: - # Anything with options always gets installed - return None - else: - return pkg_there + if wanted_pip.version is not None: + # Check if version wanted will work with whats installed + if str(wanted_pip.version) not in pip_there: + msg = ("Pip %s is already installed" + " and it is not compatible with desired" + " pip %s") + msg = msg % (pip_there, wanted_pip) + raise excp.DependencyException(msg) + return pip_there def _execute_pip(self, cmd): pip_cmd = self._get_pip_command() @@ -67,8 +78,9 @@ class Packager(pack.Packager): try: sh.execute(*pip_cmd, run_as_root=True) finally: - # The known packages installed is probably not consistent anymore so uncache it - pip_helper.uncache() + # The known packages installed is probably + # not consistent anymore so uncache it + pip_helper.Helper.uncache() def _install(self, pip): cmd = ['install'] + PIP_INSTALL_CMD_OPTS @@ -78,13 +90,14 @@ class Packager(pack.Packager): options = [str(options)] for opt in options: cmd.append(str(opt)) - cmd.append(self._make_pip_name(pip['name'], pip.get('version'))) + install_what = extract_requirement(pip) + cmd.append(str(install_what)) self._execute_pip(cmd) def _remove(self, pip): # Versions don't seem to matter here... - name = self._make_pip_name(pip['name'], None) - if not pip_helper.is_installed(self._get_pip_command(), name): + remove_what = extract_requirement(pip) + if not self.helper.is_installed(remove_what.name): return - cmd = ['uninstall'] + PIP_UNINSTALL_CMD_OPTS + [name] + cmd = ['uninstall'] + PIP_UNINSTALL_CMD_OPTS + [remove_what.name] self._execute_pip(cmd) diff --git a/anvil/packaging/rpm.py b/anvil/packaging/rpm.py index e5ab2bcb..ee7ff3a7 100644 --- a/anvil/packaging/rpm.py +++ b/anvil/packaging/rpm.py @@ -40,6 +40,7 @@ class DependencyPackager(comp.Component): self.match_installed = tu.make_bool(kargs.get('match_installed')) self._build_paths = None self._details = None + self._helper = yum_helper.Helper() @property def build_paths(self): @@ -82,9 +83,10 @@ class DependencyPackager(comp.Component): def _match_version_installed(self, yum_pkg): if not self.match_installed: return yum_pkg - installed_pkg = yum_helper.get_installed(yum_pkg['name']) - if not installed_pkg: + installed_pkgs = self._helper.get_installed(yum_pkg['name']) + if not installed_pkgs: return yum_pkg + installed_pkg = installed_pkgs[0] pkg_new = copy.deepcopy(yum_pkg) pkg_new['version'] = installed_pkg.printVer() return pkg_new diff --git a/anvil/packaging/yum.py b/anvil/packaging/yum.py index 944dd5b8..b25502c9 100644 --- a/anvil/packaging/yum.py +++ b/anvil/packaging/yum.py @@ -25,23 +25,47 @@ LOG = logging.getLogger(__name__) YUM_CMD = ['yum'] YUM_INSTALL = ["install", "-y", "-t"] YUM_REMOVE = ['erase', '-y', "-t"] -NAMED_VERSION_TEMPL = "%s-%s" + + +def extract_requirement(pkg_info): + p_name = pkg_info.get('name', '') + p_name = p_name.strip() + if not p_name: + raise ValueError("Yum requirement provided with an empty name") + p_version = pkg_info.get('version') + if p_version is not None: + if isinstance(p_version, (int, float, long)): + p_version = str(p_version) + if not isinstance(p_version, (str, basestring)): + raise TypeError("Yum requirement version must be a string or numeric type") + return yum_helper.Requirement(p_name, p_version) class YumPackager(pack.Packager): - - def _format_pkg_name(self, name, version): - if version: - return NAMED_VERSION_TEMPL % (name, version) - else: - return str(name) + def __init__(self, distro, remove_default=False): + pack.Packager.__init__(self, distro, remove_default) + self.helper = yum_helper.Helper() def _anything_there(self, pkg): - return yum_helper.get_installed(pkg['name'], pkg.get('version')) + req = extract_requirement(pkg) + whats_installed = self.helper.get_installed(req.name) + if len(whats_installed) == 0: + return None + # Check if whats installed will work, and if it won't + # then hopefully whats being installed will and + # something later doesn't come by and change it... + for p in whats_installed: + if p.verGE(req.package): + return p + # Warn that incompat. versions could be installed... + LOG.warn("There was %s matches to %s found, none satisified our request!", + len(whats_installed), req) + return None def _execute_yum(self, cmd, **kargs): yum_cmd = YUM_CMD + cmd - return sh.execute(*yum_cmd, run_as_root=True, check_exit_code=True, **kargs) + return sh.execute(*yum_cmd, run_as_root=True, + check_exit_code=True, **kargs) def _remove_special(self, name, info): return False @@ -50,18 +74,35 @@ class YumPackager(pack.Packager): return False def _install(self, pkg): - name = pkg['name'] - if self._install_special(name, pkg): + req = extract_requirement(pkg) + if self._install_special(req.name, pkg): return else: - cmd = YUM_INSTALL + [self._format_pkg_name(name, pkg.get("version"))] + cmd = YUM_INSTALL + [str(req)] self._execute_yum(cmd) def _remove(self, pkg): - name = pkg['name'] - if not yum_helper.is_installed(name, version=None): + req = extract_requirement(pkg) + whats_there = self.helper.get_installed(req.name) + matched = False + if req.version is None and len(whats_there): + # Always matches... + matched = True + else: + for p in whats_there: + if p.verEQ(req.package): + matched = True + if not len(whats_there): + # Nothing installed return - if self._remove_special(name, pkg): + if not matched: + # Warn that incompat. version could be uninstalled + LOG.warn("Removing package named %s even though %s packages with different versions exist", + req, len(whats_there)) + if self._remove_special(req.name, pkg): return - cmd = YUM_REMOVE + [self._format_pkg_name(name, pkg.get("version"))] + # Not removing specific version, this could + # cause problems but should be good enough until + # it does cause problems... + cmd = YUM_REMOVE + [req.name] self._execute_yum(cmd) diff --git a/anvil/shell.py b/anvil/shell.py index 590b6bb3..5048a032 100644 --- a/anvil/shell.py +++ b/anvil/shell.py @@ -125,8 +125,6 @@ def execute(*cmd, **kwargs): process_env = env.get() for (k, v) in env_overrides.items(): process_env[k] = str(v) - else: - process_env = env.get() # LOG.debug("With environment %s", process_env) demoter = None diff --git a/conf/distros/rhel.yaml b/conf/distros/rhel.yaml index 5df3c95b..8cf48ea1 100644 --- a/conf/distros/rhel.yaml +++ b/conf/distros/rhel.yaml @@ -223,9 +223,6 @@ components: target: "/usr/bin/sphinx-quickstart" - source: "/usr/bin/sphinx-1.0-autogen" target: "/usr/bin/sphinx-autogen" - - name: testtools - package: - name: python-testtools - name: unittest2 package: name: python-unittest2 @@ -271,6 +268,7 @@ components: # but versions are miss-matched - "-U" - name: sqlalchemy-migrate + - name: testtools # Seems like the version in rhel is to old... glance: action_classes: install: anvil.components.glance:GlanceInstaller @@ -284,6 +282,9 @@ components: # and versions inside those files into distribution # package names equivalents (if possible) pip_to_package: + - name: pyOpenSSL + package: + name: pyOpenSSL - name: jsonschema package: name: python-jsonschema @@ -383,12 +384,12 @@ components: uninstall: anvil.components.nova:NovaUninstaller packages: - name: MySQL-python + # Helpful utilities/core + # system requirements - name: dnsmasq removable: false - name: ebtables removable: false - - name: fuse - removable: false - name: iptables removable: false - name: iputils @@ -420,9 +421,12 @@ components: pips: - name: Cheetah version: "2.4.4" + - name: fixtures # Seems for testing only subsystems: compute: packages: + - name: fuse # Needed for mounting + removable: false - name: guestfish removable: false - name: iscsi-initiator-utils @@ -439,8 +443,6 @@ components: removable: false - name: libvirt-python removable: false - - name: lvm2 - removable: false - name: qemu-img removable: false - name: qemu-kvm diff --git a/conf/personas/in-a-box/basic-web.yaml b/conf/personas/in-a-box/basic-web.yaml index 9d3dacd1..483ba515 100644 --- a/conf/personas/in-a-box/basic-web.yaml +++ b/conf/personas/in-a-box/basic-web.yaml @@ -5,16 +5,17 @@ components: - db - rabbit-mq - keystone +# Client used by many components - keystone-client - glance +# Clients used by nova (+ others) - glance-client -- no-vnc - cinder-client +- quantum-client +- swift-client # Seems only needed for horizon? +- no-vnc - nova - nova-client -# These seem needed for horizon (even if not used?) -- quantum-client -- swift-client - horizon # Super client, so install after other clients - openstack-client diff --git a/conf/personas/in-a-box/basic.yaml b/conf/personas/in-a-box/basic.yaml index c3a98224..2dfff623 100644 --- a/conf/personas/in-a-box/basic.yaml +++ b/conf/personas/in-a-box/basic.yaml @@ -5,9 +5,10 @@ components: - db - rabbit-mq - keystone +# Client used by many components - keystone-client - glance -# Clients used by nova +# Clients used by nova (+ others) - glance-client - cinder-client - quantum-client