Setup Ceph default RBD features

Nova-lxd requires that our Ceph images only contain the features
supported by the kernel RBD driver, and a discussion on the dev mailing
list suggests that 1 should work fine as the driver level

This commit contains a charmhelpers sync to bring in the new
flag to support configuration sent from the ceph charms.

Change-Id: Ifb20000fb8ec7c54e752fdb3acdc16043637a81d

hooks/charmhelpers/contrib/charmsupport/nrpe.py

@@ -125,7 +125,7 @@ class CheckException(Exception):
 
 
 class Check(object):
-    shortname_re = '[A-Za-z0-9-_]+$'
+    shortname_re = '[A-Za-z0-9-_.]+$'
     service_template = ("""
 #---------------------------------------------------
 # This file is Juju managed

hooks/charmhelpers/contrib/network/ip.py

@@ -243,11 +243,11 @@ def is_ipv6_disabled():
     try:
         result = subprocess.check_output(
             ['sysctl', 'net.ipv6.conf.all.disable_ipv6'],
-            stderr=subprocess.STDOUT)
+            stderr=subprocess.STDOUT,
+            universal_newlines=True)
     except subprocess.CalledProcessError:
         return True
-    if six.PY3:
+
-        result = result.decode('UTF-8')
     return "net.ipv6.conf.all.disable_ipv6 = 1" in result
 
 

hooks/charmhelpers/contrib/openstack/context.py

@@ -97,6 +97,7 @@ from charmhelpers.contrib.openstack.utils import (
     git_determine_usr_bin,
     git_determine_python_path,
     enable_memcache,
+    snap_install_requested,
 )
 from charmhelpers.core.unitdata import kv
 
@@ -244,6 +245,11 @@ class SharedDBContext(OSContextGenerator):
                     'database_password': rdata.get(password_setting),
                     'database_type': 'mysql'
                 }
+                # Note(coreycb): We can drop mysql+pymysql if we want when the
+                # following review lands, though it seems mysql+pymysql would
+                # be preferred. https://review.openstack.org/#/c/462190/
+                if snap_install_requested():
+                    ctxt['database_type'] = 'mysql+pymysql'
                 if self.context_complete(ctxt):
                     db_ssl(rdata, ctxt, self.ssl_dir)
                     return ctxt
@@ -510,6 +516,10 @@ class CephContext(OSContextGenerator):
                     ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
                 if not ctxt.get('key'):
                     ctxt['key'] = relation_get('key', rid=rid, unit=unit)
+                if not ctxt.get('rbd_features'):
+                    default_features = relation_get('rbd-features', rid=rid, unit=unit)
+                    if default_features is not None:
+                        ctxt['rbd_features'] = default_features
 
                 ceph_addrs = relation_get('ceph-public-address', rid=rid,
                                           unit=unit)
@@ -1397,6 +1407,18 @@ class NeutronAPIContext(OSContextGenerator):
                 'rel_key': 'dns-domain',
                 'default': None,
             },
+            'polling_interval': {
+                'rel_key': 'polling-interval',
+                'default': 2,
+            },
+            'rpc_response_timeout': {
+                'rel_key': 'rpc-response-timeout',
+                'default': 60,
+            },
+            'report_interval': {
+                'rel_key': 'report-interval',
+                'default': 30,
+            },
         }
         ctxt = self.get_neutron_options({})
         for rid in relation_ids('neutron-plugin-api'):

hooks/charmhelpers/contrib/openstack/templates/ceph.conf

@@ -1,6 +1,6 @@
 ###############################################################################
 # [ WARNING ]
-# cinder configuration file maintained by Juju
+# ceph configuration file maintained by Juju
 # local changes may be overwritten.
 ###############################################################################
 [global]
@@ -12,6 +12,9 @@ mon host = {{ mon_hosts }}
 log to syslog = {{ use_syslog }}
 err to syslog = {{ use_syslog }}
 clog to syslog = {{ use_syslog }}
+{% if rbd_features %}
+rbd default features = {{ rbd_features }}
+{% endif %}
 
 [client]
 {% if rbd_client_cache_settings -%}

hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg

@@ -1,6 +1,6 @@
 global
-    log {{ local_host }} local0
+    log /var/lib/haproxy/dev/log local0
-    log {{ local_host }} local1 notice
+    log /var/lib/haproxy/dev/log local1 notice
     maxconn 20000
     user haproxy
     group haproxy

hooks/charmhelpers/contrib/openstack/templates/section-oslo-notifications

@@ -0,0 +1,8 @@
+{% if transport_url -%}
+[oslo_messaging_notifications]
+driver = messagingv2
+transport_url = {{ transport_url }}
+{% if notification_topics -%}
+topics = {{ notification_topics }}
+{% endif -%}
+{% endif -%}

hooks/charmhelpers/contrib/openstack/utils.py

@@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import (
     status_set,
     hook_name,
     application_version_set,
+    cached,
 )
 
 from charmhelpers.core.strutils import BasicStringComparator
@@ -90,6 +91,13 @@ from charmhelpers.fetch import (
     GPGKeyError,
     get_upstream_version
 )
+
+from charmhelpers.fetch.snap import (
+    snap_install,
+    snap_refresh,
+    SNAP_CHANNELS,
+)
+
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
 from charmhelpers.contrib.openstack.exceptions import OSContextError
@@ -327,8 +335,10 @@ def get_os_codename_install_source(src):
         return ca_rel
 
     # Best guess match based on deb string provided
-    if src.startswith('deb') or src.startswith('ppa'):
+    if (src.startswith('deb') or
-        for k, v in six.iteritems(OPENSTACK_CODENAMES):
+            src.startswith('ppa') or
+            src.startswith('snap')):
+        for v in OPENSTACK_CODENAMES.values():
             if v in src:
                 return v
 
@@ -397,6 +407,19 @@ def get_swift_codename(version):
 
 def get_os_codename_package(package, fatal=True):
     '''Derive OpenStack release codename from an installed package.'''
+
+    if snap_install_requested():
+        cmd = ['snap', 'list', package]
+        try:
+            out = subprocess.check_output(cmd)
+        except subprocess.CalledProcessError as e:
+            return None
+        lines = out.split('\n')
+        for line in lines:
+            if package in line:
+                # Second item in list is Version
+                return line.split()[1]
+
     import apt_pkg as apt
 
     cache = apt_cache()
@@ -613,6 +636,9 @@ def openstack_upgrade_available(package):
     import apt_pkg as apt
     src = config('openstack-origin')
     cur_vers = get_os_version_package(package)
+    if not cur_vers:
+        # The package has not been installed yet do not attempt upgrade
+        return False
     if "swift" in package:
         codename = get_os_codename_install_source(src)
         avail_vers = get_os_version_codename_swift(codename)
@@ -1863,6 +1889,30 @@ def pausable_restart_on_change(restart_map, stopstart=False,
     return wrap
 
 
+def ordered(orderme):
+    """Converts the provided dictionary into a collections.OrderedDict.
+
+    The items in the returned OrderedDict will be inserted based on the
+    natural sort order of the keys. Nested dictionaries will also be sorted
+    in order to ensure fully predictable ordering.
+
+    :param orderme: the dict to order
+    :return: collections.OrderedDict
+    :raises: ValueError: if `orderme` isn't a dict instance.
+    """
+    if not isinstance(orderme, dict):
+        raise ValueError('argument must be a dict type')
+
+    result = OrderedDict()
+    for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]):
+        if isinstance(v, dict):
+            result[k] = ordered(v)
+        else:
+            result[k] = v
+
+    return result
+
+
 def config_flags_parser(config_flags):
     """Parses config flags string into dict.
 
@@ -1874,15 +1924,13 @@ def config_flags_parser(config_flags):
          example, a string in the format of 'key1=value1, key2=value2' will
          return a dict of:
 
-             {'key1': 'value1',
+             {'key1': 'value1', 'key2': 'value2'}.
-              'key2': 'value2'}.
 
       2. A string in the above format, but supporting a comma-delimited list
          of values for the same key. For example, a string in the format of
          'key1=value1, key2=value3,value4,value5' will return a dict of:
 
-             {'key1', 'value1',
+             {'key1': 'value1', 'key2': 'value2,value3,value4'}
-              'key2', 'value2,value3,value4'}
 
       3. A string containing a colon character (:) prior to an equal
          character (=) will be treated as yaml and parsed as such. This can be
@@ -1902,7 +1950,7 @@ def config_flags_parser(config_flags):
     equals = config_flags.find('=')
     if colon > 0:
         if colon < equals or equals < 0:
-            return yaml.safe_load(config_flags)
+            return ordered(yaml.safe_load(config_flags))
 
     if config_flags.find('==') >= 0:
         juju_log("config_flags is not in expected format (key=value)",
@@ -1915,7 +1963,7 @@ def config_flags_parser(config_flags):
     # split on '='.
     split = config_flags.strip(' =').split('=')
     limit = len(split)
-    flags = {}
+    flags = OrderedDict()
     for i in range(0, limit - 1):
         current = split[i]
         next = split[i + 1]
@@ -1982,3 +2030,84 @@ def token_cache_pkgs(source=None, release=None):
     if enable_memcache(source=source, release=release):
         packages.extend(['memcached', 'python-memcache'])
     return packages
+
+
+def update_json_file(filename, items):
+    """Updates the json `filename` with a given dict.
+    :param filename: json filename (i.e.: /etc/glance/policy.json)
+    :param items: dict of items to update
+    """
+    with open(filename) as fd:
+        policy = json.load(fd)
+    policy.update(items)
+    with open(filename, "w") as fd:
+        fd.write(json.dumps(policy, indent=4))
+
+
+@cached
+def snap_install_requested():
+    """ Determine if installing from snaps
+
+    If openstack-origin is of the form snap:channel-series-release
+    and channel is in SNAPS_CHANNELS return True.
+    """
+    origin = config('openstack-origin') or ""
+    if not origin.startswith('snap:'):
+        return False
+
+    _src = origin[5:]
+    channel, series, release = _src.split('-')
+    if channel.lower() in SNAP_CHANNELS:
+        return True
+    return False
+
+
+def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
+    """Generate a dictionary of snap install information from origin
+
+    @param snaps: List of snaps
+    @param src: String of openstack-origin or source of the form
+        snap:channel-series-track
+    @param mode: String classic, devmode or jailmode
+    @returns: Dictionary of snaps with channels and modes
+    """
+
+    if not src.startswith('snap:'):
+        juju_log("Snap source is not a snap origin", 'WARN')
+        return {}
+
+    _src = src[5:]
+    _channel, _series, _release = _src.split('-')
+    channel = '--channel={}/{}'.format(_release, _channel)
+
+    return {snap: {'channel': channel, 'mode': mode}
+            for snap in snaps}
+
+
+def install_os_snaps(snaps, refresh=False):
+    """Install OpenStack snaps from channel and with mode
+
+    @param snaps: Dictionary of snaps with channels and modes of the form:
+        {'snap_name': {'channel': 'snap_channel',
+                       'mode': 'snap_mode'}}
+        Where channel a snapstore channel and mode is --classic, --devmode or
+        --jailmode.
+    @param post_snap_install: Callback function to run after snaps have been
+    installed
+    """
+
+    def _ensure_flag(flag):
+        if flag.startswith('--'):
+            return flag
+        return '--{}'.format(flag)
+
+    if refresh:
+        for snap in snaps.keys():
+            snap_refresh(snap,
+                         _ensure_flag(snaps[snap]['channel']),
+                         _ensure_flag(snaps[snap]['mode']))
+    else:
+        for snap in snaps.keys():
+            snap_install(snap,
+                         _ensure_flag(snaps[snap]['channel']),
+                         _ensure_flag(snaps[snap]['mode']))

hooks/charmhelpers/contrib/storage/linux/bcache.py

@@ -0,0 +1,74 @@
+# Copyright 2017 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import json
+
+from charmhelpers.core.hookenv import log
+
+stats_intervals = ['stats_day', 'stats_five_minute',
+                   'stats_hour', 'stats_total']
+
+SYSFS = '/sys'
+
+
+class Bcache(object):
+    """Bcache behaviour
+    """
+
+    def __init__(self, cachepath):
+        self.cachepath = cachepath
+
+    @classmethod
+    def fromdevice(cls, devname):
+        return cls('{}/block/{}/bcache'.format(SYSFS, devname))
+
+    def __str__(self):
+        return self.cachepath
+
+    def get_stats(self, interval):
+        """Get cache stats
+        """
+        intervaldir = 'stats_{}'.format(interval)
+        path = "{}/{}".format(self.cachepath, intervaldir)
+        out = dict()
+        for elem in os.listdir(path):
+            out[elem] = open('{}/{}'.format(path, elem)).read().strip()
+        return out
+
+
+def get_bcache_fs():
+    """Return all cache sets
+    """
+    cachesetroot = "{}/fs/bcache".format(SYSFS)
+    try:
+        dirs = os.listdir(cachesetroot)
+    except OSError:
+        log("No bcache fs found")
+        return []
+    cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')])
+    return cacheset
+
+
+def get_stats_action(cachespec, interval):
+    """Action for getting bcache statistics for a given cachespec.
+    Cachespec can either be a device name, eg. 'sdb', which will retrieve
+    cache stats for the given device, or 'global', which will retrieve stats
+    for all cachesets
+    """
+    if cachespec == 'global':
+        caches = get_bcache_fs()
+    else:
+        caches = [Bcache.fromdevice(cachespec)]
+    res = dict((c.cachepath, c.get_stats(interval)) for c in caches)
+    return json.dumps(res, indent=4, separators=(',', ': '))

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -1372,7 +1372,7 @@ class CephConfContext(object):
             return {}
 
         conf = config_flags_parser(conf)
-        if type(conf) != dict:
+        if not isinstance(conf, dict):
             log("Provided config-flags is not a dictionary - ignoring",
                 level=WARNING)
             return {}

hooks/charmhelpers/core/hookenv.py

@@ -202,6 +202,27 @@ def service_name():
     return local_unit().split('/')[0]
 
 
+def principal_unit():
+    """Returns the principal unit of this unit, otherwise None"""
+    # Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT
+    principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None)
+    # If it's empty, then this unit is the principal
+    if principal_unit == '':
+        return os.environ['JUJU_UNIT_NAME']
+    elif principal_unit is not None:
+        return principal_unit
+    # For Juju 2.1 and below, let's try work out the principle unit by
+    # the various charms' metadata.yaml.
+    for reltype in relation_types():
+        for rid in relation_ids(reltype):
+            for unit in related_units(rid):
+                md = _metadata_unit(unit)
+                subordinate = md.pop('subordinate', None)
+                if not subordinate:
+                    return unit
+    return None
+
+
 @cached
 def remote_service_name(relid=None):
     """The remote service name for a given relation-id (or the current relation)"""
@@ -478,6 +499,21 @@ def metadata():
         return yaml.safe_load(md)
 
 
+def _metadata_unit(unit):
+    """Given the name of a unit (e.g. apache2/0), get the unit charm's
+    metadata.yaml. Very similar to metadata() but allows us to inspect
+    other units. Unit needs to be co-located, such as a subordinate or
+    principal/primary.
+
+    :returns: metadata.yaml as a python object.
+
+    """
+    basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
+    unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
+    with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md:
+        return yaml.safe_load(md)
+
+
 @cached
 def relation_types():
     """Get a list of relation types supported by this charm"""

hooks/charmhelpers/fetch/snap.py

@@ -18,15 +18,23 @@ If writing reactive charms, use the snap layer:
 https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html
 """
 import subprocess
-from os import environ
+import os
 from time import sleep
 from charmhelpers.core.hookenv import log
 
 __author__ = 'Joseph Borg <joseph.borg@canonical.com>'
 
-SNAP_NO_LOCK = 1  # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved).
+# The return code for "couldn't acquire lock" in Snap
+# (hopefully this will be improved).
+SNAP_NO_LOCK = 1
 SNAP_NO_LOCK_RETRY_DELAY = 10  # Wait X seconds between Snap lock checks.
 SNAP_NO_LOCK_RETRY_COUNT = 30  # Retry to acquire the lock X times.
+SNAP_CHANNELS = [
+    'edge',
+    'beta',
+    'candidate',
+    'stable',
+]
 
 
 class CouldNotAcquireLockException(Exception):
@@ -47,13 +55,17 @@ def _snap_exec(commands):
 
     while return_code is None or return_code == SNAP_NO_LOCK:
         try:
-            return_code = subprocess.check_call(['snap'] + commands, env=environ)
+            return_code = subprocess.check_call(['snap'] + commands,
+                                                env=os.environ)
         except subprocess.CalledProcessError as e:
             retry_count += + 1
             if retry_count > SNAP_NO_LOCK_RETRY_COUNT:
-                raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT)
+                raise CouldNotAcquireLockException(
+                    'Could not aquire lock after {} attempts'
+                    .format(SNAP_NO_LOCK_RETRY_COUNT))
             return_code = e.returncode
-            log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN')
+            log('Snap failed to acquire lock, trying again in {} seconds.'
+                .format(SNAP_NO_LOCK_RETRY_DELAY, level='WARN'))
             sleep(SNAP_NO_LOCK_RETRY_DELAY)
 
     return return_code

hooks/charmhelpers/fetch/ubuntu.py

@@ -139,7 +139,7 @@ CLOUD_ARCHIVE_POCKETS = {
     'xenial-updates/ocata': 'xenial-updates/ocata',
     'ocata/proposed': 'xenial-proposed/ocata',
     'xenial-ocata/proposed': 'xenial-proposed/ocata',
-    'xenial-ocata/newton': 'xenial-proposed/ocata',
+    'xenial-proposed/ocata': 'xenial-proposed/ocata',
     # Pike
     'pike': 'xenial-updates/pike',
     'xenial-pike': 'xenial-updates/pike',
@@ -147,7 +147,7 @@ CLOUD_ARCHIVE_POCKETS = {
     'xenial-updates/pike': 'xenial-updates/pike',
     'pike/proposed': 'xenial-proposed/pike',
     'xenial-pike/proposed': 'xenial-proposed/pike',
-    'xenial-pike/newton': 'xenial-proposed/pike',
+    'xenial-proposed/pike': 'xenial-proposed/pike',
     # Queens
     'queens': 'xenial-updates/queens',
     'xenial-queens': 'xenial-updates/queens',
@@ -155,13 +155,13 @@ CLOUD_ARCHIVE_POCKETS = {
     'xenial-updates/queens': 'xenial-updates/queens',
     'queens/proposed': 'xenial-proposed/queens',
     'xenial-queens/proposed': 'xenial-proposed/queens',
-    'xenial-queens/newton': 'xenial-proposed/queens',
+    'xenial-proposed/queens': 'xenial-proposed/queens',
 }
 
 
 APT_NO_LOCK = 100  # The return code for "couldn't acquire lock" in APT.
 CMD_RETRY_DELAY = 10  # Wait 10 seconds between command retries.
-CMD_RETRY_COUNT = 30  # Retry a failing fatal command X times.
+CMD_RETRY_COUNT = 3  # Retry a failing fatal command X times.
 
 
 def filter_installed_packages(packages):
@@ -364,6 +364,7 @@ def add_source(source, key=None, fail_invalid=False):
         (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging),
         (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check),
         (r"^cloud:(.*)$", _add_cloud_pocket),
+        (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check),
     ])
     if source is None:
         source = ''

tests/charmhelpers/contrib/network/ip.py

@@ -243,11 +243,11 @@ def is_ipv6_disabled():
     try:
         result = subprocess.check_output(
             ['sysctl', 'net.ipv6.conf.all.disable_ipv6'],
-            stderr=subprocess.STDOUT)
+            stderr=subprocess.STDOUT,
+            universal_newlines=True)
     except subprocess.CalledProcessError:
         return True
-    if six.PY3:
+
-        result = result.decode('UTF-8')
     return "net.ipv6.conf.all.disable_ipv6 = 1" in result
 
 

tests/charmhelpers/contrib/openstack/utils.py

@@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import (
     status_set,
     hook_name,
     application_version_set,
+    cached,
 )
 
 from charmhelpers.core.strutils import BasicStringComparator
@@ -90,6 +91,13 @@ from charmhelpers.fetch import (
     GPGKeyError,
     get_upstream_version
 )
+
+from charmhelpers.fetch.snap import (
+    snap_install,
+    snap_refresh,
+    SNAP_CHANNELS,
+)
+
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
 from charmhelpers.contrib.openstack.exceptions import OSContextError
@@ -327,8 +335,10 @@ def get_os_codename_install_source(src):
         return ca_rel
 
     # Best guess match based on deb string provided
-    if src.startswith('deb') or src.startswith('ppa'):
+    if (src.startswith('deb') or
-        for k, v in six.iteritems(OPENSTACK_CODENAMES):
+            src.startswith('ppa') or
+            src.startswith('snap')):
+        for v in OPENSTACK_CODENAMES.values():
             if v in src:
                 return v
 
@@ -397,6 +407,19 @@ def get_swift_codename(version):
 
 def get_os_codename_package(package, fatal=True):
     '''Derive OpenStack release codename from an installed package.'''
+
+    if snap_install_requested():
+        cmd = ['snap', 'list', package]
+        try:
+            out = subprocess.check_output(cmd)
+        except subprocess.CalledProcessError as e:
+            return None
+        lines = out.split('\n')
+        for line in lines:
+            if package in line:
+                # Second item in list is Version
+                return line.split()[1]
+
     import apt_pkg as apt
 
     cache = apt_cache()
@@ -613,6 +636,9 @@ def openstack_upgrade_available(package):
     import apt_pkg as apt
     src = config('openstack-origin')
     cur_vers = get_os_version_package(package)
+    if not cur_vers:
+        # The package has not been installed yet do not attempt upgrade
+        return False
     if "swift" in package:
         codename = get_os_codename_install_source(src)
         avail_vers = get_os_version_codename_swift(codename)
@@ -1863,6 +1889,30 @@ def pausable_restart_on_change(restart_map, stopstart=False,
     return wrap
 
 
+def ordered(orderme):
+    """Converts the provided dictionary into a collections.OrderedDict.
+
+    The items in the returned OrderedDict will be inserted based on the
+    natural sort order of the keys. Nested dictionaries will also be sorted
+    in order to ensure fully predictable ordering.
+
+    :param orderme: the dict to order
+    :return: collections.OrderedDict
+    :raises: ValueError: if `orderme` isn't a dict instance.
+    """
+    if not isinstance(orderme, dict):
+        raise ValueError('argument must be a dict type')
+
+    result = OrderedDict()
+    for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]):
+        if isinstance(v, dict):
+            result[k] = ordered(v)
+        else:
+            result[k] = v
+
+    return result
+
+
 def config_flags_parser(config_flags):
     """Parses config flags string into dict.
 
@@ -1874,15 +1924,13 @@ def config_flags_parser(config_flags):
          example, a string in the format of 'key1=value1, key2=value2' will
          return a dict of:
 
-             {'key1': 'value1',
+             {'key1': 'value1', 'key2': 'value2'}.
-              'key2': 'value2'}.
 
       2. A string in the above format, but supporting a comma-delimited list
          of values for the same key. For example, a string in the format of
          'key1=value1, key2=value3,value4,value5' will return a dict of:
 
-             {'key1', 'value1',
+             {'key1': 'value1', 'key2': 'value2,value3,value4'}
-              'key2', 'value2,value3,value4'}
 
       3. A string containing a colon character (:) prior to an equal
          character (=) will be treated as yaml and parsed as such. This can be
@@ -1902,7 +1950,7 @@ def config_flags_parser(config_flags):
     equals = config_flags.find('=')
     if colon > 0:
         if colon < equals or equals < 0:
-            return yaml.safe_load(config_flags)
+            return ordered(yaml.safe_load(config_flags))
 
     if config_flags.find('==') >= 0:
         juju_log("config_flags is not in expected format (key=value)",
@@ -1915,7 +1963,7 @@ def config_flags_parser(config_flags):
     # split on '='.
     split = config_flags.strip(' =').split('=')
     limit = len(split)
-    flags = {}
+    flags = OrderedDict()
     for i in range(0, limit - 1):
         current = split[i]
         next = split[i + 1]
@@ -1982,3 +2030,84 @@ def token_cache_pkgs(source=None, release=None):
     if enable_memcache(source=source, release=release):
         packages.extend(['memcached', 'python-memcache'])
     return packages
+
+
+def update_json_file(filename, items):
+    """Updates the json `filename` with a given dict.
+    :param filename: json filename (i.e.: /etc/glance/policy.json)
+    :param items: dict of items to update
+    """
+    with open(filename) as fd:
+        policy = json.load(fd)
+    policy.update(items)
+    with open(filename, "w") as fd:
+        fd.write(json.dumps(policy, indent=4))
+
+
+@cached
+def snap_install_requested():
+    """ Determine if installing from snaps
+
+    If openstack-origin is of the form snap:channel-series-release
+    and channel is in SNAPS_CHANNELS return True.
+    """
+    origin = config('openstack-origin') or ""
+    if not origin.startswith('snap:'):
+        return False
+
+    _src = origin[5:]
+    channel, series, release = _src.split('-')
+    if channel.lower() in SNAP_CHANNELS:
+        return True
+    return False
+
+
+def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
+    """Generate a dictionary of snap install information from origin
+
+    @param snaps: List of snaps
+    @param src: String of openstack-origin or source of the form
+        snap:channel-series-track
+    @param mode: String classic, devmode or jailmode
+    @returns: Dictionary of snaps with channels and modes
+    """
+
+    if not src.startswith('snap:'):
+        juju_log("Snap source is not a snap origin", 'WARN')
+        return {}
+
+    _src = src[5:]
+    _channel, _series, _release = _src.split('-')
+    channel = '--channel={}/{}'.format(_release, _channel)
+
+    return {snap: {'channel': channel, 'mode': mode}
+            for snap in snaps}
+
+
+def install_os_snaps(snaps, refresh=False):
+    """Install OpenStack snaps from channel and with mode
+
+    @param snaps: Dictionary of snaps with channels and modes of the form:
+        {'snap_name': {'channel': 'snap_channel',
+                       'mode': 'snap_mode'}}
+        Where channel a snapstore channel and mode is --classic, --devmode or
+        --jailmode.
+    @param post_snap_install: Callback function to run after snaps have been
+    installed
+    """
+
+    def _ensure_flag(flag):
+        if flag.startswith('--'):
+            return flag
+        return '--{}'.format(flag)
+
+    if refresh:
+        for snap in snaps.keys():
+            snap_refresh(snap,
+                         _ensure_flag(snaps[snap]['channel']),
+                         _ensure_flag(snaps[snap]['mode']))
+    else:
+        for snap in snaps.keys():
+            snap_install(snap,
+                         _ensure_flag(snaps[snap]['channel']),
+                         _ensure_flag(snaps[snap]['mode']))

tests/charmhelpers/contrib/storage/linux/bcache.py

@@ -0,0 +1,74 @@
+# Copyright 2017 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import json
+
+from charmhelpers.core.hookenv import log
+
+stats_intervals = ['stats_day', 'stats_five_minute',
+                   'stats_hour', 'stats_total']
+
+SYSFS = '/sys'
+
+
+class Bcache(object):
+    """Bcache behaviour
+    """
+
+    def __init__(self, cachepath):
+        self.cachepath = cachepath
+
+    @classmethod
+    def fromdevice(cls, devname):
+        return cls('{}/block/{}/bcache'.format(SYSFS, devname))
+
+    def __str__(self):
+        return self.cachepath
+
+    def get_stats(self, interval):
+        """Get cache stats
+        """
+        intervaldir = 'stats_{}'.format(interval)
+        path = "{}/{}".format(self.cachepath, intervaldir)
+        out = dict()
+        for elem in os.listdir(path):
+            out[elem] = open('{}/{}'.format(path, elem)).read().strip()
+        return out
+
+
+def get_bcache_fs():
+    """Return all cache sets
+    """
+    cachesetroot = "{}/fs/bcache".format(SYSFS)
+    try:
+        dirs = os.listdir(cachesetroot)
+    except OSError:
+        log("No bcache fs found")
+        return []
+    cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')])
+    return cacheset
+
+
+def get_stats_action(cachespec, interval):
+    """Action for getting bcache statistics for a given cachespec.
+    Cachespec can either be a device name, eg. 'sdb', which will retrieve
+    cache stats for the given device, or 'global', which will retrieve stats
+    for all cachesets
+    """
+    if cachespec == 'global':
+        caches = get_bcache_fs()
+    else:
+        caches = [Bcache.fromdevice(cachespec)]
+    res = dict((c.cachepath, c.get_stats(interval)) for c in caches)
+    return json.dumps(res, indent=4, separators=(',', ': '))

tests/charmhelpers/contrib/storage/linux/ceph.py

@@ -1372,7 +1372,7 @@ class CephConfContext(object):
             return {}
 
         conf = config_flags_parser(conf)
-        if type(conf) != dict:
+        if not isinstance(conf, dict):
             log("Provided config-flags is not a dictionary - ignoring",
                 level=WARNING)
             return {}

tests/charmhelpers/core/hookenv.py

@@ -202,6 +202,27 @@ def service_name():
     return local_unit().split('/')[0]
 
 
+def principal_unit():
+    """Returns the principal unit of this unit, otherwise None"""
+    # Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT
+    principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None)
+    # If it's empty, then this unit is the principal
+    if principal_unit == '':
+        return os.environ['JUJU_UNIT_NAME']
+    elif principal_unit is not None:
+        return principal_unit
+    # For Juju 2.1 and below, let's try work out the principle unit by
+    # the various charms' metadata.yaml.
+    for reltype in relation_types():
+        for rid in relation_ids(reltype):
+            for unit in related_units(rid):
+                md = _metadata_unit(unit)
+                subordinate = md.pop('subordinate', None)
+                if not subordinate:
+                    return unit
+    return None
+
+
 @cached
 def remote_service_name(relid=None):
     """The remote service name for a given relation-id (or the current relation)"""
@@ -478,6 +499,21 @@ def metadata():
         return yaml.safe_load(md)
 
 
+def _metadata_unit(unit):
+    """Given the name of a unit (e.g. apache2/0), get the unit charm's
+    metadata.yaml. Very similar to metadata() but allows us to inspect
+    other units. Unit needs to be co-located, such as a subordinate or
+    principal/primary.
+
+    :returns: metadata.yaml as a python object.
+
+    """
+    basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
+    unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
+    with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md:
+        return yaml.safe_load(md)
+
+
 @cached
 def relation_types():
     """Get a list of relation types supported by this charm"""

tests/charmhelpers/fetch/snap.py

@@ -18,15 +18,23 @@ If writing reactive charms, use the snap layer:
 https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html
 """
 import subprocess
-from os import environ
+import os
 from time import sleep
 from charmhelpers.core.hookenv import log
 
 __author__ = 'Joseph Borg <joseph.borg@canonical.com>'
 
-SNAP_NO_LOCK = 1  # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved).
+# The return code for "couldn't acquire lock" in Snap
+# (hopefully this will be improved).
+SNAP_NO_LOCK = 1
 SNAP_NO_LOCK_RETRY_DELAY = 10  # Wait X seconds between Snap lock checks.
 SNAP_NO_LOCK_RETRY_COUNT = 30  # Retry to acquire the lock X times.
+SNAP_CHANNELS = [
+    'edge',
+    'beta',
+    'candidate',
+    'stable',
+]
 
 
 class CouldNotAcquireLockException(Exception):
@@ -47,13 +55,17 @@ def _snap_exec(commands):
 
     while return_code is None or return_code == SNAP_NO_LOCK:
         try:
-            return_code = subprocess.check_call(['snap'] + commands, env=environ)
+            return_code = subprocess.check_call(['snap'] + commands,
+                                                env=os.environ)
         except subprocess.CalledProcessError as e:
             retry_count += + 1
             if retry_count > SNAP_NO_LOCK_RETRY_COUNT:
-                raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT)
+                raise CouldNotAcquireLockException(
+                    'Could not aquire lock after {} attempts'
+                    .format(SNAP_NO_LOCK_RETRY_COUNT))
             return_code = e.returncode
-            log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN')
+            log('Snap failed to acquire lock, trying again in {} seconds.'
+                .format(SNAP_NO_LOCK_RETRY_DELAY, level='WARN'))
             sleep(SNAP_NO_LOCK_RETRY_DELAY)
 
     return return_code

tests/charmhelpers/fetch/ubuntu.py

@@ -139,7 +139,7 @@ CLOUD_ARCHIVE_POCKETS = {
     'xenial-updates/ocata': 'xenial-updates/ocata',
     'ocata/proposed': 'xenial-proposed/ocata',
     'xenial-ocata/proposed': 'xenial-proposed/ocata',
-    'xenial-ocata/newton': 'xenial-proposed/ocata',
+    'xenial-proposed/ocata': 'xenial-proposed/ocata',
     # Pike
     'pike': 'xenial-updates/pike',
     'xenial-pike': 'xenial-updates/pike',
@@ -147,7 +147,7 @@ CLOUD_ARCHIVE_POCKETS = {
     'xenial-updates/pike': 'xenial-updates/pike',
     'pike/proposed': 'xenial-proposed/pike',
     'xenial-pike/proposed': 'xenial-proposed/pike',
-    'xenial-pike/newton': 'xenial-proposed/pike',
+    'xenial-proposed/pike': 'xenial-proposed/pike',
     # Queens
     'queens': 'xenial-updates/queens',
     'xenial-queens': 'xenial-updates/queens',
@@ -155,13 +155,13 @@ CLOUD_ARCHIVE_POCKETS = {
     'xenial-updates/queens': 'xenial-updates/queens',
     'queens/proposed': 'xenial-proposed/queens',
     'xenial-queens/proposed': 'xenial-proposed/queens',
-    'xenial-queens/newton': 'xenial-proposed/queens',
+    'xenial-proposed/queens': 'xenial-proposed/queens',
 }
 
 
 APT_NO_LOCK = 100  # The return code for "couldn't acquire lock" in APT.
 CMD_RETRY_DELAY = 10  # Wait 10 seconds between command retries.
-CMD_RETRY_COUNT = 30  # Retry a failing fatal command X times.
+CMD_RETRY_COUNT = 3  # Retry a failing fatal command X times.
 
 
 def filter_installed_packages(packages):
@@ -364,6 +364,7 @@ def add_source(source, key=None, fail_invalid=False):
         (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging),
         (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check),
         (r"^cloud:(.*)$", _add_cloud_pocket),
+        (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check),
     ])
     if source is None:
         source = ''