Use the pip code to get an initial pip installed list and same with rpms via yum code

anvil/components/__init__.py

@@ -43,11 +43,12 @@ from anvil import exceptions as excp
 from anvil import importer
 from anvil import log as logging
 from anvil import packager
-from anvil import pip
 from anvil import shell as sh
 from anvil import trace as tr
 from anvil import utils
 
+from anvil.packaging import pip
+
 LOG = logging.getLogger(__name__)
 
 #### 
@@ -63,6 +64,14 @@ class ProgramStatus(object):
         self.status = status
         self.details = details
 
+####
+#### Utils...
+####
+
+def make_packager(package, distro, default_class):
+    cls = packager.get_packager_class(package, default_class)
+    return cls(distro)
+
 
 #### 
 #### INSTALL CLASSES
@@ -72,7 +81,6 @@ class PkgInstallComponent(component.Component):
     def __init__(self, *args, **kargs):
         component.Component.__init__(self, *args, **kargs)
         self.tracewriter = tr.TraceWriter(self.trace_files['install'], break_if_there=False)
-        self.package_registries = kargs.get('package_registries', {})
 
     def _get_download_config(self):
         return None
@@ -127,12 +135,6 @@ class PkgInstallComponent(component.Component):
         pkg_list = self._clear_package_duplicates(pkg_list)
         return pkg_list
 
-    def _make_packager(self, name, pkg_info, default_cls):
-        if name not in self.package_registries:
-            self.package_registries[name] = packager.Registry()
-        cls = packager.get_packager_class(pkg_info, default_cls)
-        return cls(self.distro, self.package_registries[name])
-
     def install(self):
         LOG.debug('Preparing to install packages for: %r', self.name)
         pkgs = self.packages
@@ -142,7 +144,7 @@ class PkgInstallComponent(component.Component):
                 header="Setting up %s distribution packages" % (len(pkg_names)))
             with utils.progress_bar('Installing', len(pkgs)) as p_bar:
                 for (i, p) in enumerate(pkgs):
-                    installer = self._make_packager('distro', p, self.distro.package_manager_class)
+                    installer = make_packager(p, self.distro, self.distro.package_manager_class)
                     self.tracewriter.package_installed(p)
                     installer.install(p)
                     p_bar.update(i + 1)
@@ -150,13 +152,13 @@ class PkgInstallComponent(component.Component):
     def pre_install(self):
         pkgs = self.packages
         for p in pkgs:
-            installer = self._make_packager('distro', p, self.distro.package_manager_class)
+            installer = make_packager(p, self.distro, self.distro.package_manager_class)
             installer.pre_install(p, self.params)
 
     def post_install(self):
         pkgs = self.packages
         for p in pkgs:
-            installer = self._make_packager('distro', p, self.distro.package_manager_class)
+            installer = make_packager(p, self.distro, self.distro.package_manager_class)
             installer.post_install(p, self.params)
 
     @property
@@ -378,7 +380,7 @@ class PythonInstallComponent(PkgInstallComponent):
             with utils.progress_bar('Installing', len(pips)) as p_bar:
                 for (i, p) in enumerate(pips):
                     self.tracewriter.pip_installed(p)
-                    installer = self._make_packager('pip', p, pip.Packager)
+                    installer = make_packager(p, self.distro, pip.Packager)
                     installer.install(p)
                     p_bar.update(i + 1)
 
@@ -416,13 +418,13 @@ class PythonInstallComponent(PkgInstallComponent):
         self._verify_pip_requires()
         PkgInstallComponent.pre_install(self)
         for p in self.pips:
-            installer = self._make_packager('pip', p, pip.Packager)
+            installer = make_packager(p, self.distro, pip.Packager)
             installer.pre_install(p, self.params)
 
     def post_install(self):
         PkgInstallComponent.post_install(self)
         for p in self.pips:
-            installer = self._make_packager('pip', p, pip.Packager)
+            installer = make_packager(p, self.distro, pip.Packager)
             installer.post_install(p, self.params)
 
     def _install_python_setups(self):
@@ -630,13 +632,6 @@ class PkgUninstallComponent(component.Component):
     def __init__(self, *args, **kargs):
         component.Component.__init__(self, *args, **kargs)
         self.tracereader = tr.TraceReader(self.trace_files['install'])
-        self.package_registries = kargs.get('package_registries', {})
-
-    def _make_packager(self, name, pkg_info, default_cls):
-        if name not in self.package_registries:
-            self.package_registries[name] = packager.Registry()
-        cls = packager.get_packager_class(pkg_info, default_cls)
-        return cls(self.distro, self.package_registries[name])
 
     def unconfigure(self):
         self._unconfigure_files()
@@ -680,7 +675,7 @@ class PkgUninstallComponent(component.Component):
             which_removed = set()
             with utils.progress_bar('Uninstalling', len(pkgs), reverse=True) as p_bar:
                 for (i, p) in enumerate(pkgs):
-                    uninstaller = self._make_packager('distro', p, self.distro.package_manager_class)
+                    uninstaller = make_packager(p, self.distro, self.distro.package_manager_class)
                     if uninstaller.remove(p):
                         which_removed.add(p['name'])
                     p_bar.update(i + 1)
@@ -731,7 +726,7 @@ class PythonUninstallComponent(PkgUninstallComponent):
             with utils.progress_bar('Uninstalling', len(pips), reverse=True) as p_bar:
                 for (i, p) in enumerate(pips):
                     try:
-                        uninstaller = self._make_packager('pip', p, pip.Packager)
+                        uninstaller = make_packager(p, self.distro, pip.Packager)
                         uninstaller.remove(p)
                     except excp.ProcessExecutionError as e:
                         # NOTE(harlowja): pip seems to die if a pkg isn't there even in quiet mode

anvil/components/keystone.py

@@ -212,7 +212,9 @@ class KeystoneRuntime(comp.PythonRuntime):
                                                             **self.get_option('glance'))
             params['nova'] = nhelper.get_shared_params(ip=self.get_option('ip'),
                                                           **self.get_option('nova'))
-            init_what = utils.load_yaml_text(utils.expand_template(sh.load_file(fn), params))
+            init_what = utils.load_yaml(sh.load_file(fn))
+            
+            init_what = utils.expand_template_deep(init_what, params)
             khelper.Initializer(params['keystone']['service_token'],
                                 params['keystone']['endpoints']['admin']['uri']).initialize(**init_what)
             # Writing this makes sure that we don't init again

anvil/distros/rhel.py

@@ -191,9 +191,8 @@ class YumPackagerWithRelinks(yum.YumPackager):
             if not tgt or not src:
                 continue
             src = glob.glob(src)
-            tgt = glob.glob(tgt)
+            if not isinstance(tgt, (list, tuple)):
-            if not tgt:
+                tgt = [tgt]
-                tgt = [entry.get('target')]
             if len(src) != len(tgt):
                 raise RuntimeError("Unable to link %s sources to %s locations" % (len(src), len(tgt)))
             for i in range(len(src)):

anvil/packager.py

@@ -15,7 +15,6 @@
 #    under the License.
 
 import abc
-import pkg_resources
 
 from anvil import exceptions as excp
 from anvil import colorizer
@@ -29,7 +28,6 @@ VERSION_CHARS = ['=', '>', "<"]
 
 
 class NullVersion(object):
-
     def __init__(self, name):
         self.name = name
 
@@ -37,36 +35,26 @@ class NullVersion(object):
         return True
 
     def __str__(self):
-        return "%s (no version)" % (self.name)
+        return "%s (unknown version)" % (self.name)
 
 
 class Registry(object):
-
     def __init__(self):
         self.installed = {}
         self.removed = {}
 
 
 class Packager(object):
-
     __meta__ = abc.ABCMeta
 
-    def __init__(self, distro, registry):
+    def __init__(self, distro, registry=None):
         self.distro = distro
+        if not registry:
+            registry = Registry()
         self.registry = registry
 
     def _parse_version(self, name, version):
-        if version:
+        return NullVersion(name)
-            # This won't work for all package versions (ie crazy names)
-            # but good enough for now...
-            if contains_version_check(version):
-                full_name = "%s%s" % (name, version)
-            else:
-                full_name = "%s==%s" % (name, version)
-            p_version = pkg_resources.Requirement.parse(full_name)
-        else:
-            p_version = NullVersion(name)
-        return p_version
 
     def _compare_against_installed(self, incoming_version, installed_version):
         if not incoming_version and installed_version:
@@ -76,11 +64,6 @@ class Packager(object):
             # Assume whats installed will work
             # (not really the case all the time)
             return True
-        if contains_version_check(incoming_version):
-            cleaned_version = incoming_version
-            for c in VERSION_CHARS:
-                cleaned_version = cleaned_version.replace(c, '')
-            return self._compare_against_installed(cleaned_version.strip(), installed_version)
         if not incoming_version in installed_version:
             # Not in the range of the installed version (bad!)
             return False
@@ -150,5 +133,4 @@ def get_packager_class(package_info, default_packager_class=None):
     packager_name = packager_name.strip()
     if not packager_name:
         return default_packager_class
-    packager_class = importer.import_entry_point(packager_name)
+    return importer.import_entry_point(packager_name)
-    return packager_class

anvil/packaging/helpers/__init__.py

@@ -0,0 +1,15 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

anvil/packaging/helpers/pip_helper.py

@@ -0,0 +1,41 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from anvil import log as logging
+from anvil import packager as pack
+from anvil import utils
+
+import pip
+from pip.util import get_installed_distributions
+
+LOG = logging.getLogger(__name__)
+
+
+def make_registry():
+    installations = {}
+    for dist in get_installed_distributions(local_only=True):
+        freq = pip.FrozenRequirement.from_dist(dist, [])
+        if freq.req and freq.name:
+            name = freq.name.lower()
+            installations[name] = freq.req
+    # TODO(harlowja) use the pip version/requirement to enhance this...
+    reg = pack.Registry()
+    for (name, _req) in installations.items():
+        reg.installed[name] = pack.NullVersion(name)
+    LOG.debug("Identified %s packages already installed by pip", len(reg.installed))
+    utils.log_object(reg.installed, logger=LOG, level=logging.DEBUG)
+    return reg

anvil/packaging/helpers/yum_helper.py

@@ -0,0 +1,35 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from anvil import log as logging
+from anvil import packager as pack
+from anvil import utils
+
+from yum import YumBase
+
+LOG = logging.getLogger(__name__)
+
+
+def make_registry():
+    reg = pack.Registry()
+    yb = YumBase()
+    yb.conf.cache = False
+    for p in yb.rpmdb.returnPackages():
+        # TODO(harlowja) use the rpm version comparision to enhance this...
+        reg.installed[p.name] = pack.NullVersion(p.name)
+    LOG.debug("Identified %s packages already installed by yum", len(reg.installed))
+    utils.log_object(reg.installed, logger=LOG, level=logging.DEBUG)
+    return reg

anvil/packaging/pip.py

@@ -19,6 +19,8 @@ from anvil import log as logging
 from anvil import shell as sh
 from anvil import packager as pack
 
+from anvil.packaging.helpers import pip_helper
+
 LOG = logging.getLogger(__name__)
 
 PIP_UNINSTALL_CMD_OPTS = ['-y', '-q']
@@ -26,6 +28,10 @@ PIP_INSTALL_CMD_OPTS = ['-q']
 
 
 class Packager(pack.Packager):
+    PIP_REGISTRY = pip_helper.make_registry()
+
+    def __init__(self, distro):
+        pack.Packager.__init__(self, distro, Packager.PIP_REGISTRY)
 
     def _make_pip_name(self, name, version):
         if version is None:
@@ -35,6 +41,18 @@ class Packager(pack.Packager):
         else:
             return "%s==%s" % (name, version)
 
+    def _parse_version(self, name, version):
+        if version:
+            # This should work for all pip packages
+            if contains_version_check(version):
+                full_name = "%s%s" % (name, version)
+            else:
+                full_name = "%s==%s" % (name, version)
+            p_version = pkg_resources.Requirement.parse(full_name)
+        else:
+            p_version = pack.Packager._parse_version(self, name, version)
+        return p_version
+
     def _get_pip_command(self):
         return self.distro.get_command_config('pip')
 

anvil/packaging/yum.py

@@ -18,6 +18,8 @@ from anvil import log as logging
 from anvil import packager as pack
 from anvil import shell as sh
 
+from anvil.packaging.helpers import yum_helper
+
 LOG = logging.getLogger(__name__)
 
 # Root yum command
@@ -34,6 +36,10 @@ VERSION_TEMPL = "%s-%s"
 
 
 class YumPackager(pack.Packager):
+    YUM_REGISTRY = yum_helper.make_registry()
+
+    def __init__(self, distro):
+        pack.Packager.__init__(self, distro, YumPackager.YUM_REGISTRY)
 
     def _format_pkg_name(self, name, version):
         if version:
@@ -43,9 +49,7 @@ class YumPackager(pack.Packager):
 
     def _execute_yum(self, cmd, **kargs):
         full_cmd = YUM_CMD + cmd
-        return sh.execute(*full_cmd, run_as_root=True,
+        return sh.execute(*full_cmd, run_as_root=True, check_exit_code=True, **kargs)
-            check_exit_code=True,
-            **kargs)
 
     def _remove_special(self, name, info):
         return False

anvil/utils.py

@@ -74,6 +74,27 @@ def expand_template(contents, params):
     return Template(str(contents), searchList=[params]).respond()
 
 
+def expand_template_deep(root, params):
+    if isinstance(root, (basestring, str)):
+        return expand_template(root, params)
+    if isinstance(root, (list, tuple)):
+        n_list = []
+        for i in root:
+            n_list.append(expand_template_deep(i, params))
+        return n_list
+    if isinstance(root, (dict)):
+        n_dict = {}
+        for (k, v) in root.items():
+            n_dict[k] = expand_template_deep(v, params)
+        return n_dict
+    if isinstance(root, (set)):
+        n_set = set()
+        for v in root:
+            n_set.add(expand_template_deep(v, params))
+        return n_set
+    return root
+
+
 def load_yaml(fn):
     return load_yaml_text(sh.load_file(fn))
 

conf/templates/keystone/init_what.yaml

@@ -1,5 +1,6 @@
 ##
-## This is a cheetah/yaml template!
+## This is a yaml template (with cheetah template 
+## strings that will be filled in)...
 ##
 ---
 endpoints: