From 19a8a1d8b55d15e9fb6a2ad194a1e965a92bca90 Mon Sep 17 00:00:00 2001 From: harlowja Date: Sat, 25 Aug 2012 18:33:06 -0700 Subject: [PATCH] Use the pip code to get an initial pip installed list and same with rpms via yum code --- anvil/components/__init__.py | 41 +++++++++++--------------- anvil/components/keystone.py | 4 ++- anvil/distros/rhel.py | 5 ++-- anvil/packager.py | 30 ++++--------------- anvil/packaging/helpers/__init__.py | 15 ++++++++++ anvil/packaging/helpers/pip_helper.py | 41 ++++++++++++++++++++++++++ anvil/packaging/helpers/yum_helper.py | 35 ++++++++++++++++++++++ anvil/{ => packaging}/pip.py | 18 +++++++++++ anvil/packaging/yum.py | 10 +++++-- anvil/utils.py | 21 +++++++++++++ conf/templates/keystone/init_what.yaml | 3 +- 11 files changed, 168 insertions(+), 55 deletions(-) create mode 100644 anvil/packaging/helpers/__init__.py create mode 100644 anvil/packaging/helpers/pip_helper.py create mode 100644 anvil/packaging/helpers/yum_helper.py rename anvil/{ => packaging}/pip.py (76%) diff --git a/anvil/components/__init__.py b/anvil/components/__init__.py index 10650d02..c07fe91c 100644 --- a/anvil/components/__init__.py +++ b/anvil/components/__init__.py @@ -43,11 +43,12 @@ from anvil import exceptions as excp from anvil import importer from anvil import log as logging from anvil import packager -from anvil import pip from anvil import shell as sh from anvil import trace as tr from anvil import utils +from anvil.packaging import pip + LOG = logging.getLogger(__name__) #### @@ -63,6 +64,14 @@ class ProgramStatus(object): self.status = status self.details = details +#### +#### Utils... +#### + +def make_packager(package, distro, default_class): + cls = packager.get_packager_class(package, default_class) + return cls(distro) + #### #### INSTALL CLASSES @@ -72,7 +81,6 @@ class PkgInstallComponent(component.Component): def __init__(self, *args, **kargs): component.Component.__init__(self, *args, **kargs) self.tracewriter = tr.TraceWriter(self.trace_files['install'], break_if_there=False) - self.package_registries = kargs.get('package_registries', {}) def _get_download_config(self): return None @@ -127,12 +135,6 @@ class PkgInstallComponent(component.Component): pkg_list = self._clear_package_duplicates(pkg_list) return pkg_list - def _make_packager(self, name, pkg_info, default_cls): - if name not in self.package_registries: - self.package_registries[name] = packager.Registry() - cls = packager.get_packager_class(pkg_info, default_cls) - return cls(self.distro, self.package_registries[name]) - def install(self): LOG.debug('Preparing to install packages for: %r', self.name) pkgs = self.packages @@ -142,7 +144,7 @@ class PkgInstallComponent(component.Component): header="Setting up %s distribution packages" % (len(pkg_names))) with utils.progress_bar('Installing', len(pkgs)) as p_bar: for (i, p) in enumerate(pkgs): - installer = self._make_packager('distro', p, self.distro.package_manager_class) + installer = make_packager(p, self.distro, self.distro.package_manager_class) self.tracewriter.package_installed(p) installer.install(p) p_bar.update(i + 1) @@ -150,13 +152,13 @@ class PkgInstallComponent(component.Component): def pre_install(self): pkgs = self.packages for p in pkgs: - installer = self._make_packager('distro', p, self.distro.package_manager_class) + installer = make_packager(p, self.distro, self.distro.package_manager_class) installer.pre_install(p, self.params) def post_install(self): pkgs = self.packages for p in pkgs: - installer = self._make_packager('distro', p, self.distro.package_manager_class) + installer = make_packager(p, self.distro, self.distro.package_manager_class) installer.post_install(p, self.params) @property @@ -378,7 +380,7 @@ class PythonInstallComponent(PkgInstallComponent): with utils.progress_bar('Installing', len(pips)) as p_bar: for (i, p) in enumerate(pips): self.tracewriter.pip_installed(p) - installer = self._make_packager('pip', p, pip.Packager) + installer = make_packager(p, self.distro, pip.Packager) installer.install(p) p_bar.update(i + 1) @@ -416,13 +418,13 @@ class PythonInstallComponent(PkgInstallComponent): self._verify_pip_requires() PkgInstallComponent.pre_install(self) for p in self.pips: - installer = self._make_packager('pip', p, pip.Packager) + installer = make_packager(p, self.distro, pip.Packager) installer.pre_install(p, self.params) def post_install(self): PkgInstallComponent.post_install(self) for p in self.pips: - installer = self._make_packager('pip', p, pip.Packager) + installer = make_packager(p, self.distro, pip.Packager) installer.post_install(p, self.params) def _install_python_setups(self): @@ -630,13 +632,6 @@ class PkgUninstallComponent(component.Component): def __init__(self, *args, **kargs): component.Component.__init__(self, *args, **kargs) self.tracereader = tr.TraceReader(self.trace_files['install']) - self.package_registries = kargs.get('package_registries', {}) - - def _make_packager(self, name, pkg_info, default_cls): - if name not in self.package_registries: - self.package_registries[name] = packager.Registry() - cls = packager.get_packager_class(pkg_info, default_cls) - return cls(self.distro, self.package_registries[name]) def unconfigure(self): self._unconfigure_files() @@ -680,7 +675,7 @@ class PkgUninstallComponent(component.Component): which_removed = set() with utils.progress_bar('Uninstalling', len(pkgs), reverse=True) as p_bar: for (i, p) in enumerate(pkgs): - uninstaller = self._make_packager('distro', p, self.distro.package_manager_class) + uninstaller = make_packager(p, self.distro, self.distro.package_manager_class) if uninstaller.remove(p): which_removed.add(p['name']) p_bar.update(i + 1) @@ -731,7 +726,7 @@ class PythonUninstallComponent(PkgUninstallComponent): with utils.progress_bar('Uninstalling', len(pips), reverse=True) as p_bar: for (i, p) in enumerate(pips): try: - uninstaller = self._make_packager('pip', p, pip.Packager) + uninstaller = make_packager(p, self.distro, pip.Packager) uninstaller.remove(p) except excp.ProcessExecutionError as e: # NOTE(harlowja): pip seems to die if a pkg isn't there even in quiet mode diff --git a/anvil/components/keystone.py b/anvil/components/keystone.py index 438780fa..1f2a279e 100644 --- a/anvil/components/keystone.py +++ b/anvil/components/keystone.py @@ -212,7 +212,9 @@ class KeystoneRuntime(comp.PythonRuntime): **self.get_option('glance')) params['nova'] = nhelper.get_shared_params(ip=self.get_option('ip'), **self.get_option('nova')) - init_what = utils.load_yaml_text(utils.expand_template(sh.load_file(fn), params)) + init_what = utils.load_yaml(sh.load_file(fn)) + + init_what = utils.expand_template_deep(init_what, params) khelper.Initializer(params['keystone']['service_token'], params['keystone']['endpoints']['admin']['uri']).initialize(**init_what) # Writing this makes sure that we don't init again diff --git a/anvil/distros/rhel.py b/anvil/distros/rhel.py index 34c133a5..546e69bc 100644 --- a/anvil/distros/rhel.py +++ b/anvil/distros/rhel.py @@ -191,9 +191,8 @@ class YumPackagerWithRelinks(yum.YumPackager): if not tgt or not src: continue src = glob.glob(src) - tgt = glob.glob(tgt) - if not tgt: - tgt = [entry.get('target')] + if not isinstance(tgt, (list, tuple)): + tgt = [tgt] if len(src) != len(tgt): raise RuntimeError("Unable to link %s sources to %s locations" % (len(src), len(tgt))) for i in range(len(src)): diff --git a/anvil/packager.py b/anvil/packager.py index bbb023d5..c617db63 100644 --- a/anvil/packager.py +++ b/anvil/packager.py @@ -15,7 +15,6 @@ # under the License. import abc -import pkg_resources from anvil import exceptions as excp from anvil import colorizer @@ -29,7 +28,6 @@ VERSION_CHARS = ['=', '>', "<"] class NullVersion(object): - def __init__(self, name): self.name = name @@ -37,36 +35,26 @@ class NullVersion(object): return True def __str__(self): - return "%s (no version)" % (self.name) + return "%s (unknown version)" % (self.name) class Registry(object): - def __init__(self): self.installed = {} self.removed = {} class Packager(object): - __meta__ = abc.ABCMeta - def __init__(self, distro, registry): + def __init__(self, distro, registry=None): self.distro = distro + if not registry: + registry = Registry() self.registry = registry def _parse_version(self, name, version): - if version: - # This won't work for all package versions (ie crazy names) - # but good enough for now... - if contains_version_check(version): - full_name = "%s%s" % (name, version) - else: - full_name = "%s==%s" % (name, version) - p_version = pkg_resources.Requirement.parse(full_name) - else: - p_version = NullVersion(name) - return p_version + return NullVersion(name) def _compare_against_installed(self, incoming_version, installed_version): if not incoming_version and installed_version: @@ -76,11 +64,6 @@ class Packager(object): # Assume whats installed will work # (not really the case all the time) return True - if contains_version_check(incoming_version): - cleaned_version = incoming_version - for c in VERSION_CHARS: - cleaned_version = cleaned_version.replace(c, '') - return self._compare_against_installed(cleaned_version.strip(), installed_version) if not incoming_version in installed_version: # Not in the range of the installed version (bad!) return False @@ -150,5 +133,4 @@ def get_packager_class(package_info, default_packager_class=None): packager_name = packager_name.strip() if not packager_name: return default_packager_class - packager_class = importer.import_entry_point(packager_name) - return packager_class + return importer.import_entry_point(packager_name) diff --git a/anvil/packaging/helpers/__init__.py b/anvil/packaging/helpers/__init__.py new file mode 100644 index 00000000..a7bfb005 --- /dev/null +++ b/anvil/packaging/helpers/__init__.py @@ -0,0 +1,15 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/anvil/packaging/helpers/pip_helper.py b/anvil/packaging/helpers/pip_helper.py new file mode 100644 index 00000000..8f1f6150 --- /dev/null +++ b/anvil/packaging/helpers/pip_helper.py @@ -0,0 +1,41 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from anvil import log as logging +from anvil import packager as pack +from anvil import utils + +import pip +from pip.util import get_installed_distributions + +LOG = logging.getLogger(__name__) + + +def make_registry(): + installations = {} + for dist in get_installed_distributions(local_only=True): + freq = pip.FrozenRequirement.from_dist(dist, []) + if freq.req and freq.name: + name = freq.name.lower() + installations[name] = freq.req + # TODO(harlowja) use the pip version/requirement to enhance this... + reg = pack.Registry() + for (name, _req) in installations.items(): + reg.installed[name] = pack.NullVersion(name) + LOG.debug("Identified %s packages already installed by pip", len(reg.installed)) + utils.log_object(reg.installed, logger=LOG, level=logging.DEBUG) + return reg diff --git a/anvil/packaging/helpers/yum_helper.py b/anvil/packaging/helpers/yum_helper.py new file mode 100644 index 00000000..80b7971c --- /dev/null +++ b/anvil/packaging/helpers/yum_helper.py @@ -0,0 +1,35 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from anvil import log as logging +from anvil import packager as pack +from anvil import utils + +from yum import YumBase + +LOG = logging.getLogger(__name__) + + +def make_registry(): + reg = pack.Registry() + yb = YumBase() + yb.conf.cache = False + for p in yb.rpmdb.returnPackages(): + # TODO(harlowja) use the rpm version comparision to enhance this... + reg.installed[p.name] = pack.NullVersion(p.name) + LOG.debug("Identified %s packages already installed by yum", len(reg.installed)) + utils.log_object(reg.installed, logger=LOG, level=logging.DEBUG) + return reg diff --git a/anvil/pip.py b/anvil/packaging/pip.py similarity index 76% rename from anvil/pip.py rename to anvil/packaging/pip.py index 327a9adb..2ad3e959 100644 --- a/anvil/pip.py +++ b/anvil/packaging/pip.py @@ -19,6 +19,8 @@ from anvil import log as logging from anvil import shell as sh from anvil import packager as pack +from anvil.packaging.helpers import pip_helper + LOG = logging.getLogger(__name__) PIP_UNINSTALL_CMD_OPTS = ['-y', '-q'] @@ -26,6 +28,10 @@ PIP_INSTALL_CMD_OPTS = ['-q'] class Packager(pack.Packager): + PIP_REGISTRY = pip_helper.make_registry() + + def __init__(self, distro): + pack.Packager.__init__(self, distro, Packager.PIP_REGISTRY) def _make_pip_name(self, name, version): if version is None: @@ -35,6 +41,18 @@ class Packager(pack.Packager): else: return "%s==%s" % (name, version) + def _parse_version(self, name, version): + if version: + # This should work for all pip packages + if contains_version_check(version): + full_name = "%s%s" % (name, version) + else: + full_name = "%s==%s" % (name, version) + p_version = pkg_resources.Requirement.parse(full_name) + else: + p_version = pack.Packager._parse_version(self, name, version) + return p_version + def _get_pip_command(self): return self.distro.get_command_config('pip') diff --git a/anvil/packaging/yum.py b/anvil/packaging/yum.py index 8ecd942c..23d71ee0 100644 --- a/anvil/packaging/yum.py +++ b/anvil/packaging/yum.py @@ -18,6 +18,8 @@ from anvil import log as logging from anvil import packager as pack from anvil import shell as sh +from anvil.packaging.helpers import yum_helper + LOG = logging.getLogger(__name__) # Root yum command @@ -34,6 +36,10 @@ VERSION_TEMPL = "%s-%s" class YumPackager(pack.Packager): + YUM_REGISTRY = yum_helper.make_registry() + + def __init__(self, distro): + pack.Packager.__init__(self, distro, YumPackager.YUM_REGISTRY) def _format_pkg_name(self, name, version): if version: @@ -43,9 +49,7 @@ class YumPackager(pack.Packager): def _execute_yum(self, cmd, **kargs): full_cmd = YUM_CMD + cmd - return sh.execute(*full_cmd, run_as_root=True, - check_exit_code=True, - **kargs) + return sh.execute(*full_cmd, run_as_root=True, check_exit_code=True, **kargs) def _remove_special(self, name, info): return False diff --git a/anvil/utils.py b/anvil/utils.py index 66bea398..9da7844b 100644 --- a/anvil/utils.py +++ b/anvil/utils.py @@ -74,6 +74,27 @@ def expand_template(contents, params): return Template(str(contents), searchList=[params]).respond() +def expand_template_deep(root, params): + if isinstance(root, (basestring, str)): + return expand_template(root, params) + if isinstance(root, (list, tuple)): + n_list = [] + for i in root: + n_list.append(expand_template_deep(i, params)) + return n_list + if isinstance(root, (dict)): + n_dict = {} + for (k, v) in root.items(): + n_dict[k] = expand_template_deep(v, params) + return n_dict + if isinstance(root, (set)): + n_set = set() + for v in root: + n_set.add(expand_template_deep(v, params)) + return n_set + return root + + def load_yaml(fn): return load_yaml_text(sh.load_file(fn)) diff --git a/conf/templates/keystone/init_what.yaml b/conf/templates/keystone/init_what.yaml index 65d7fc9c..a8dc16ed 100644 --- a/conf/templates/keystone/init_what.yaml +++ b/conf/templates/keystone/init_what.yaml @@ -1,5 +1,6 @@ ## -## This is a cheetah/yaml template! +## This is a yaml template (with cheetah template +## strings that will be filled in)... ## --- endpoints: