Sync charm helpers

hooks/charmhelpers/contrib/openstack/utils.py

@@ -23,7 +23,7 @@ from charmhelpers.contrib.storage.linux.lvm import (
 )
 
 from charmhelpers.core.host import lsb_release, mounts, umount
-from charmhelpers.fetch import apt_install
+from charmhelpers.fetch import apt_install, apt_cache
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
 
@@ -134,13 +134,8 @@ def get_os_version_codename(codename):
 def get_os_codename_package(package, fatal=True):
     '''Derive OpenStack release codename from an installed package.'''
     import apt_pkg as apt
-    apt.init()
 
-    # Tell apt to build an in-memory cache to prevent race conditions (if
+    cache = apt_cache()
-    # another process is already building the cache).
-    apt.config.set("Dir::Cache::pkgcache", "")
-
-    cache = apt.Cache()
 
     try:
         pkg = cache[package]

hooks/charmhelpers/core/hookenv.py

@@ -285,8 +285,9 @@ def relation_get(attribute=None, unit=None, rid=None):
         raise
 
 
-def relation_set(relation_id=None, relation_settings={}, **kwargs):
+def relation_set(relation_id=None, relation_settings=None, **kwargs):
     """Set relation information for the current unit"""
+    relation_settings = relation_settings if relation_settings else {}
     relation_cmd_line = ['relation-set']
     if relation_id is not None:
         relation_cmd_line.extend(('-r', relation_id))

hooks/charmhelpers/core/host.py

@@ -332,13 +332,9 @@ def cmp_pkgrevno(package, revno, pkgcache=None):
 
     '''
     import apt_pkg
+    from charmhelpers.fetch import apt_cache
     if not pkgcache:
-        apt_pkg.init()
+        pkgcache = apt_cache()
-        # Force Apt to build its cache in memory. That way we avoid race
-        # conditions with other applications building the cache in the same
-        # place.
-        apt_pkg.config.set("Dir::Cache::pkgcache", "")
-        pkgcache = apt_pkg.Cache()
     pkg = pkgcache[package]
     return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
 

hooks/charmhelpers/core/services/base.py

@@ -17,20 +17,13 @@ class ServiceManager(object):
         """
         Register a list of services, given their definitions.
 
-        Traditional charm authoring is focused on implementing hooks.  That is,
-        the charm author is thinking in terms of "What hook am I handling; what
-        does this hook need to do?"  However, in most cases, the real question
-        should be "Do I have the information I need to configure and start this
-        piece of software and, if so, what are the steps for doing so?"  The
-        ServiceManager framework tries to bring the focus to the data and the
-        setup tasks, in the most declarative way possible.
-
         Service definitions are dicts in the following formats (all keys except
         'service' are optional)::
 
             {
                 "service": <service name>,
                 "required_data": <list of required data contexts>,
+                "provided_data": <list of provided data contexts>,
                 "data_ready": <one or more callbacks>,
                 "data_lost": <one or more callbacks>,
                 "start": <one or more callbacks>,
@@ -44,6 +37,10 @@ class ServiceManager(object):
         of 'data_ready' and 'start' callbacks executed.  See `is_ready()` for more
         information.
 
+        The 'provided_data' list should contain relation data providers, most likely
+        a subclass of :class:`charmhelpers.core.services.helpers.RelationContext`,
+        that will indicate a set of data to set on a given relation.
+
         The 'data_ready' value should be either a single callback, or a list of
         callbacks, to be called when all items in 'required_data' pass `is_ready()`.
         Each callback will be called with the service name as the only parameter.
@@ -123,12 +120,20 @@ class ServiceManager(object):
             self.reconfigure_services()
 
     def provide_data(self):
+        """
+        Set the relation data for each provider in the ``provided_data`` list.
+
+        A provider must have a `name` attribute, which indicates which relation
+        to set data on, and a `provide_data()` method, which returns a dict of
+        data to set.
+        """
         hook_name = hookenv.hook_name()
         for service in self.services.values():
             for provider in service.get('provided_data', []):
                 if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
                     data = provider.provide_data()
-                    if provider._is_ready(data):
+                    _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data
+                    if _ready:
                         hookenv.relation_set(None, data)
 
     def reconfigure_services(self, *service_names):

hooks/charmhelpers/fetch/__init__.py

@@ -1,4 +1,5 @@
 import importlib
+from tempfile import NamedTemporaryFile
 import time
 from yaml import safe_load
 from charmhelpers.core.host import (
@@ -116,15 +117,7 @@ class BaseFetchHandler(object):
 
 def filter_installed_packages(packages):
     """Returns a list of packages that require installation"""
-    import apt_pkg
+    cache = apt_cache()
-    apt_pkg.init()
-
-    # Tell apt to build an in-memory cache to prevent race conditions (if
-    # another process is already building the cache).
-    apt_pkg.config.set("Dir::Cache::pkgcache", "")
-    apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
-
-    cache = apt_pkg.Cache()
     _pkgs = []
     for package in packages:
         try:
@@ -137,6 +130,16 @@ def filter_installed_packages(packages):
     return _pkgs
 
 
+def apt_cache(in_memory=True):
+    """Build and return an apt cache"""
+    import apt_pkg
+    apt_pkg.init()
+    if in_memory:
+        apt_pkg.config.set("Dir::Cache::pkgcache", "")
+        apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
+    return apt_pkg.Cache()
+
+
 def apt_install(packages, options=None, fatal=False):
     """Install one or more packages"""
     if options is None:
@@ -202,6 +205,27 @@ def apt_hold(packages, fatal=False):
 
 
 def add_source(source, key=None):
+    """Add a package source to this system.
+
+    @param source: a URL or sources.list entry, as supported by
+    add-apt-repository(1). Examples:
+        ppa:charmers/example
+        deb https://stub:key@private.example.com/ubuntu trusty main
+
+    In addition:
+        'proposed:' may be used to enable the standard 'proposed'
+        pocket for the release.
+        'cloud:' may be used to activate official cloud archive pockets,
+        such as 'cloud:icehouse'
+
+    @param key: A key to be added to the system's APT keyring and used
+    to verify the signatures on packages. Ideally, this should be an
+    ASCII format GPG public key including the block headers. A GPG key
+    id may also be used, but be aware that only insecure protocols are
+    available to retrieve the actual public key from a public keyserver
+    placing your Juju environment at risk. ppa and cloud archive keys
+    are securely added automtically, so sould not be provided.
+    """
     if source is None:
         log('Source is not present. Skipping')
         return
@@ -226,10 +250,23 @@ def add_source(source, key=None):
         release = lsb_release()['DISTRIB_CODENAME']
         with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
             apt.write(PROPOSED_POCKET.format(release))
+    else:
+        raise SourceConfigError("Unknown source: {!r}".format(source))
+
     if key:
-        subprocess.check_call(['apt-key', 'adv', '--keyserver',
+        if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
-                               'hkp://keyserver.ubuntu.com:80', '--recv',
+            with NamedTemporaryFile() as key_file:
-                               key])
+                key_file.write(key)
+                key_file.flush()
+                key_file.seek(0)
+                subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
+        else:
+            # Note that hkp: is in no way a secure protocol. Using a
+            # GPG key id is pointless from a security POV unless you
+            # absolutely trust your network and DNS.
+            subprocess.check_call(['apt-key', 'adv', '--keyserver',
+                                   'hkp://keyserver.ubuntu.com:80', '--recv',
+                                   key])
 
 
 def configure_sources(update=False,
@@ -239,7 +276,8 @@ def configure_sources(update=False,
     Configure multiple sources from charm configuration.
 
     The lists are encoded as yaml fragments in the configuration.
-    The frament needs to be included as a string.
+    The frament needs to be included as a string. Sources and their
+    corresponding keys are of the types supported by add_source().
 
     Example config:
         install_sources: |