Sync charm-helpers to ensure Rocky support

Change-Id: Id7b920ba756d4a3d2082e838c3309726ba3de278

charmhelpers/contrib/charmsupport/nrpe.py

@@ -410,16 +410,21 @@ def add_init_service_checks(nrpe, services, unit_name, immediate_check=True):
                 os.chmod(checkpath, 0o644)
 
 
-def copy_nrpe_checks():
+def copy_nrpe_checks(nrpe_files_dir=None):
     """
     Copy the nrpe checks into place
 
     """
     NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins'
-    nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks',
+    default_nrpe_files_dir = os.path.join(
-                                  'charmhelpers', 'contrib', 'openstack',
+        os.getenv('CHARM_DIR'),
+        'hooks',
+        'charmhelpers',
+        'contrib',
+        'openstack',
         'files')
-
+    if not nrpe_files_dir:
+        nrpe_files_dir = default_nrpe_files_dir
     if not os.path.exists(NAGIOS_PLUGINS):
         os.makedirs(NAGIOS_PLUGINS)
     for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")):

charmhelpers/contrib/hahelpers/apache.py

@@ -77,7 +77,7 @@ def get_ca_cert():
 def retrieve_ca_cert(cert_file):
     cert = None
     if os.path.isfile(cert_file):
-        with open(cert_file, 'r') as crt:
+        with open(cert_file, 'rb') as crt:
             cert = crt.read()
     return cert
 

charmhelpers/contrib/hahelpers/cluster.py

@@ -223,6 +223,11 @@ def https():
         return True
     if config_get('ssl_cert') and config_get('ssl_key'):
         return True
+    for r_id in relation_ids('certificates'):
+        for unit in relation_list(r_id):
+            ca = relation_get('ca', rid=r_id, unit=unit)
+            if ca:
+                return True
     for r_id in relation_ids('identity-service'):
         for unit in relation_list(r_id):
             # TODO - needs fixing for new helper as ssl_cert/key suffixes with CN

charmhelpers/contrib/openstack/amulet/deployment.py

@@ -291,6 +291,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('zesty', None): self.zesty_ocata,
             ('artful', None): self.artful_pike,
             ('bionic', None): self.bionic_queens,
+            ('bionic', 'cloud:bionic-rocky'): self.bionic_rocky,
+            ('cosmic', None): self.cosmic_rocky,
         }
         return releases[(self.series, self.openstack)]
 
@@ -306,6 +308,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('zesty', 'ocata'),
             ('artful', 'pike'),
             ('bionic', 'queens'),
+            ('cosmic', 'rocky'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

charmhelpers/contrib/openstack/amulet/utils.py

@@ -40,6 +40,7 @@ import novaclient
 import pika
 import swiftclient
 
+from charmhelpers.core.decorators import retry_on_exception
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
@@ -55,7 +56,7 @@ OPENSTACK_RELEASES_PAIRS = [
     'trusty_mitaka', 'xenial_mitaka', 'xenial_newton',
     'yakkety_newton', 'xenial_ocata', 'zesty_ocata',
     'xenial_pike', 'artful_pike', 'xenial_queens',
-    'bionic_queens']
+    'bionic_queens', 'bionic_rocky', 'cosmic_rocky']
 
 
 class OpenStackAmuletUtils(AmuletUtils):
@@ -423,6 +424,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
+    @retry_on_exception(num_retries=5, base_delay=1)
     def keystone_wait_for_propagation(self, sentry_relation_pairs,
                                       api_version):
         """Iterate over list of sentry and relation tuples and verify that
@@ -542,7 +544,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         return ep
 
     def get_default_keystone_session(self, keystone_sentry,
-                                     openstack_release=None):
+                                     openstack_release=None, api_version=2):
         """Return a keystone session object and client object assuming standard
            default settings
 
@@ -557,12 +559,12 @@ class OpenStackAmuletUtils(AmuletUtils):
                eyc
         """
         self.log.debug('Authenticating keystone admin...')
-        api_version = 2
-        client_class = keystone_client.Client
         # 11 => xenial_queens
-        if openstack_release and openstack_release >= 11:
+        if api_version == 3 or (openstack_release and openstack_release >= 11):
-            api_version = 3
             client_class = keystone_client_v3.Client
+            api_version = 3
+        else:
+            client_class = keystone_client.Client
         keystone_ip = keystone_sentry.info['public-address']
         session, auth = self.get_keystone_session(
             keystone_ip,

charmhelpers/contrib/openstack/cert_utils.py

@@ -0,0 +1,227 @@
+# Copyright 2014-2018 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Common python helper functions used for OpenStack charm certificats.
+
+import os
+import json
+
+from charmhelpers.contrib.network.ip import (
+    get_hostname,
+    resolve_network_cidr,
+)
+from charmhelpers.core.hookenv import (
+    local_unit,
+    network_get_primary_address,
+    config,
+    relation_get,
+    unit_get,
+    NoNetworkBinding,
+    log,
+    WARNING,
+)
+from charmhelpers.contrib.openstack.ip import (
+    ADMIN,
+    resolve_address,
+    get_vip_in_network,
+    INTERNAL,
+    PUBLIC,
+    ADDRESS_MAP)
+
+from charmhelpers.core.host import (
+    mkdir,
+    write_file,
+)
+
+from charmhelpers.contrib.hahelpers.apache import (
+    install_ca_cert
+)
+
+
+class CertRequest(object):
+
+    """Create a request for certificates to be generated
+    """
+
+    def __init__(self, json_encode=True):
+        self.entries = []
+        self.hostname_entry = None
+        self.json_encode = json_encode
+
+    def add_entry(self, net_type, cn, addresses):
+        """Add a request to the batch
+
+        :param net_type: str netwrok space name request is for
+        :param cn: str Canonical Name for certificate
+        :param addresses: [] List of addresses to be used as SANs
+        """
+        self.entries.append({
+            'cn': cn,
+            'addresses': addresses})
+
+    def add_hostname_cn(self):
+        """Add a request for the hostname of the machine"""
+        ip = unit_get('private-address')
+        addresses = [ip]
+        # If a vip is being used without os-hostname config or
+        # network spaces then we need to ensure the local units
+        # cert has the approriate vip in the SAN list
+        vip = get_vip_in_network(resolve_network_cidr(ip))
+        if vip:
+            addresses.append(vip)
+        self.hostname_entry = {
+            'cn': get_hostname(ip),
+            'addresses': addresses}
+
+    def add_hostname_cn_ip(self, addresses):
+        """Add an address to the SAN list for the hostname request
+
+        :param addr: [] List of address to be added
+        """
+        for addr in addresses:
+            if addr not in self.hostname_entry['addresses']:
+                self.hostname_entry['addresses'].append(addr)
+
+    def get_request(self):
+        """Generate request from the batched up entries
+
+        """
+        if self.hostname_entry:
+            self.entries.append(self.hostname_entry)
+        request = {}
+        for entry in self.entries:
+            sans = sorted(list(set(entry['addresses'])))
+            request[entry['cn']] = {'sans': sans}
+        if self.json_encode:
+            return {'cert_requests': json.dumps(request, sort_keys=True)}
+        else:
+            return {'cert_requests': request}
+
+
+def get_certificate_request(json_encode=True):
+    """Generate a certificatee requests based on the network confioguration
+
+    """
+    req = CertRequest(json_encode=json_encode)
+    req.add_hostname_cn()
+    # Add os-hostname entries
+    for net_type in [INTERNAL, ADMIN, PUBLIC]:
+        net_config = config(ADDRESS_MAP[net_type]['override'])
+        try:
+            net_addr = resolve_address(endpoint_type=net_type)
+            ip = network_get_primary_address(
+                ADDRESS_MAP[net_type]['binding'])
+            addresses = [net_addr, ip]
+            vip = get_vip_in_network(resolve_network_cidr(ip))
+            if vip:
+                addresses.append(vip)
+            if net_config:
+                req.add_entry(
+                    net_type,
+                    net_config,
+                    addresses)
+            else:
+                # There is network address with no corresponding hostname.
+                # Add the ip to the hostname cert to allow for this.
+                req.add_hostname_cn_ip(addresses)
+        except NoNetworkBinding:
+            log("Skipping request for certificate for ip in {} space, no "
+                "local address found".format(net_type), WARNING)
+    return req.get_request()
+
+
+def create_ip_cert_links(ssl_dir, custom_hostname_link=None):
+    """Create symlinks for SAN records
+
+    :param ssl_dir: str Directory to create symlinks in
+    :param custom_hostname_link: str Additional link to be created
+    """
+    hostname = get_hostname(unit_get('private-address'))
+    hostname_cert = os.path.join(
+        ssl_dir,
+        'cert_{}'.format(hostname))
+    hostname_key = os.path.join(
+        ssl_dir,
+        'key_{}'.format(hostname))
+    # Add links to hostname cert, used if os-hostname vars not set
+    for net_type in [INTERNAL, ADMIN, PUBLIC]:
+        try:
+            addr = resolve_address(endpoint_type=net_type)
+            cert = os.path.join(ssl_dir, 'cert_{}'.format(addr))
+            key = os.path.join(ssl_dir, 'key_{}'.format(addr))
+            if os.path.isfile(hostname_cert) and not os.path.isfile(cert):
+                os.symlink(hostname_cert, cert)
+                os.symlink(hostname_key, key)
+        except NoNetworkBinding:
+            log("Skipping creating cert symlink for ip in {} space, no "
+                "local address found".format(net_type), WARNING)
+    if custom_hostname_link:
+        custom_cert = os.path.join(
+            ssl_dir,
+            'cert_{}'.format(custom_hostname_link))
+        custom_key = os.path.join(
+            ssl_dir,
+            'key_{}'.format(custom_hostname_link))
+        if os.path.isfile(hostname_cert) and not os.path.isfile(custom_cert):
+            os.symlink(hostname_cert, custom_cert)
+            os.symlink(hostname_key, custom_key)
+
+
+def install_certs(ssl_dir, certs, chain=None):
+    """Install the certs passed into the ssl dir and append the chain if
+       provided.
+
+    :param ssl_dir: str Directory to create symlinks in
+    :param certs: {} {'cn': {'cert': 'CERT', 'key': 'KEY'}}
+    :param chain: str Chain to be appended to certs
+    """
+    for cn, bundle in certs.items():
+        cert_filename = 'cert_{}'.format(cn)
+        key_filename = 'key_{}'.format(cn)
+        cert_data = bundle['cert']
+        if chain:
+            # Append chain file so that clients that trust the root CA will
+            # trust certs signed by an intermediate in the chain
+            cert_data = cert_data + chain
+        write_file(
+            path=os.path.join(ssl_dir, cert_filename),
+            content=cert_data, perms=0o640)
+        write_file(
+            path=os.path.join(ssl_dir, key_filename),
+            content=bundle['key'], perms=0o640)
+
+
+def process_certificates(service_name, relation_id, unit,
+                         custom_hostname_link=None):
+    """Process the certificates supplied down the relation
+
+    :param service_name: str Name of service the certifcates are for.
+    :param relation_id: str Relation id providing the certs
+    :param unit: str Unit providing the certs
+    :param custom_hostname_link: str Name of custom link to create
+    """
+    data = relation_get(rid=relation_id, unit=unit)
+    ssl_dir = os.path.join('/etc/apache2/ssl/', service_name)
+    mkdir(path=ssl_dir)
+    name = local_unit().replace('/', '_')
+    certs = data.get('{}.processed_requests'.format(name))
+    chain = data.get('chain')
+    ca = data.get('ca')
+    if certs:
+        certs = json.loads(certs)
+        install_ca_cert(ca.encode())
+        install_certs(ssl_dir, certs, chain)
+        create_ip_cert_links(
+            ssl_dir,
+            custom_hostname_link=custom_hostname_link)

charmhelpers/contrib/openstack/context.py

@@ -190,8 +190,8 @@ class OSContextGenerator(object):
 class SharedDBContext(OSContextGenerator):
     interfaces = ['shared-db']
 
-    def __init__(self,
+    def __init__(self, database=None, user=None, relation_prefix=None,
-                 database=None, user=None, relation_prefix=None, ssl_dir=None):
+                 ssl_dir=None, relation_id=None):
         """Allows inspecting relation for settings prefixed with
         relation_prefix. This is useful for parsing access for multiple
         databases returned via the shared-db interface (eg, nova_password,
@@ -202,6 +202,7 @@ class SharedDBContext(OSContextGenerator):
         self.user = user
         self.ssl_dir = ssl_dir
         self.rel_name = self.interfaces[0]
+        self.relation_id = relation_id
 
     def __call__(self):
         self.database = self.database or config('database')
@@ -235,7 +236,12 @@ class SharedDBContext(OSContextGenerator):
         if self.relation_prefix:
             password_setting = self.relation_prefix + '_password'
 
-        for rid in relation_ids(self.interfaces[0]):
+        if self.relation_id:
+            rids = [self.relation_id]
+        else:
+            rids = relation_ids(self.interfaces[0])
+
+        for rid in rids:
             self.related = True
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
@@ -448,11 +454,13 @@ class IdentityCredentialsContext(IdentityServiceContext):
 
 class AMQPContext(OSContextGenerator):
 
-    def __init__(self, ssl_dir=None, rel_name='amqp', relation_prefix=None):
+    def __init__(self, ssl_dir=None, rel_name='amqp', relation_prefix=None,
+                 relation_id=None):
         self.ssl_dir = ssl_dir
         self.rel_name = rel_name
         self.relation_prefix = relation_prefix
         self.interfaces = [rel_name]
+        self.relation_id = relation_id
 
     def __call__(self):
         log('Generating template context for amqp', level=DEBUG)
@@ -473,7 +481,11 @@ class AMQPContext(OSContextGenerator):
             raise OSContextError
 
         ctxt = {}
-        for rid in relation_ids(self.rel_name):
+        if self.relation_id:
+            rids = [self.relation_id]
+        else:
+            rids = relation_ids(self.rel_name)
+        for rid in rids:
             ha_vip_only = False
             self.related = True
             transport_hosts = None
@@ -789,6 +801,7 @@ class ApacheSSLContext(OSContextGenerator):
         ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace)
         mkdir(path=ssl_dir)
         cert, key = get_cert(cn)
+        if cert and key:
             if cn:
                 cert_filename = 'cert_{}'.format(cn)
                 key_filename = 'key_{}'.format(cn)
@@ -797,9 +810,9 @@ class ApacheSSLContext(OSContextGenerator):
                 key_filename = 'key'
 
             write_file(path=os.path.join(ssl_dir, cert_filename),
-                   content=b64decode(cert))
+                       content=b64decode(cert), perms=0o640)
             write_file(path=os.path.join(ssl_dir, key_filename),
-                   content=b64decode(key))
+                       content=b64decode(key), perms=0o640)
 
     def configure_ca(self):
         ca_cert = get_ca_cert()
@@ -871,13 +884,21 @@ class ApacheSSLContext(OSContextGenerator):
         if not self.external_ports or not https():
             return {}
 
+        use_keystone_ca = True
+        for rid in relation_ids('certificates'):
+            if related_units(rid):
+                use_keystone_ca = False
+
+        if use_keystone_ca:
             self.configure_ca()
+
         self.enable_modules()
 
         ctxt = {'namespace': self.service_namespace,
                 'endpoints': [],
                 'ext_ports': []}
 
+        if use_keystone_ca:
             cns = self.canonical_names()
             if cns:
                 for cn in cns:
@@ -1873,10 +1894,11 @@ class EnsureDirContext(OSContextGenerator):
     context is needed to do that before rendering a template.
    '''
 
-    def __init__(self, dirname):
+    def __init__(self, dirname, **kwargs):
         '''Used merely to ensure that a given directory exists.'''
         self.dirname = dirname
+        self.kwargs = kwargs
 
     def __call__(self):
-        mkdir(self.dirname)
+        mkdir(self.dirname, **self.kwargs)
         return {}

charmhelpers/contrib/openstack/ip.py

@@ -184,3 +184,13 @@ def resolve_address(endpoint_type=PUBLIC, override=True):
                          "clustered=%s)" % (net_type, clustered))
 
     return resolved_address
+
+
+def get_vip_in_network(network):
+    matching_vip = None
+    vips = config('vip')
+    if vips:
+        for vip in vips.split():
+            if is_address_in_network(network, vip):
+                matching_vip = vip
+    return matching_vip

charmhelpers/contrib/openstack/templates/section-oslo-middleware

@@ -0,0 +1,5 @@
+[oslo_middleware]
+
+# Bug #1758675
+enable_proxy_headers_parsing = true
+

charmhelpers/contrib/openstack/templates/section-oslo-notifications

@@ -5,4 +5,7 @@ transport_url = {{ transport_url }}
 {% if notification_topics -%}
 topics = {{ notification_topics }}
 {% endif -%}
+{% if notification_format -%}
+notification_format = {{ notification_format }}
+{% endif -%}
 {% endif -%}

charmhelpers/contrib/openstack/utils.py

@@ -133,6 +133,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('zesty', 'ocata'),
     ('artful', 'pike'),
     ('bionic', 'queens'),
+    ('cosmic', 'rocky'),
 ])
 
 
@@ -151,6 +152,7 @@ OPENSTACK_CODENAMES = OrderedDict([
     ('2017.1', 'ocata'),
     ('2017.2', 'pike'),
     ('2018.1', 'queens'),
+    ('2018.2', 'rocky'),
 ])
 
 # The ugly duckling - must list releases oldest to newest
@@ -183,6 +185,8 @@ SWIFT_CODENAMES = OrderedDict([
         ['2.13.0', '2.15.0']),
     ('queens',
         ['2.16.0', '2.17.0']),
+    ('rocky',
+        ['2.18.0']),
 ])
 
 # >= Liberty version->codename mapping
@@ -306,7 +310,7 @@ def get_os_codename_install_source(src):
 
     if src.startswith('cloud:'):
         ca_rel = src.split(':')[1]
-        ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0]
+        ca_rel = ca_rel.split('-')[1].split('/')[0]
         return ca_rel
 
     # Best guess match based on deb string provided

charmhelpers/contrib/openstack/vaultlocker.py

@@ -0,0 +1,126 @@
+# Copyright 2018 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+import charmhelpers.contrib.openstack.alternatives as alternatives
+import charmhelpers.contrib.openstack.context as context
+
+import charmhelpers.core.hookenv as hookenv
+import charmhelpers.core.host as host
+import charmhelpers.core.templating as templating
+import charmhelpers.core.unitdata as unitdata
+
+VAULTLOCKER_BACKEND = 'charm-vaultlocker'
+
+
+class VaultKVContext(context.OSContextGenerator):
+    """Vault KV context for interaction with vault-kv interfaces"""
+    interfaces = ['secrets-storage']
+
+    def __init__(self, secret_backend=None):
+        super(context.OSContextGenerator, self).__init__()
+        self.secret_backend = (
+            secret_backend or 'charm-{}'.format(hookenv.service_name())
+        )
+
+    def __call__(self):
+        db = unitdata.kv()
+        last_token = db.get('last-token')
+        secret_id = db.get('secret-id')
+        for relation_id in hookenv.relation_ids(self.interfaces[0]):
+            for unit in hookenv.related_units(relation_id):
+                data = hookenv.relation_get(unit=unit,
+                                            rid=relation_id)
+                vault_url = data.get('vault_url')
+                role_id = data.get('{}_role_id'.format(hookenv.local_unit()))
+                token = data.get('{}_token'.format(hookenv.local_unit()))
+
+                if all([vault_url, role_id, token]):
+                    token = json.loads(token)
+                    vault_url = json.loads(vault_url)
+
+                    # Tokens may change when secret_id's are being
+                    # reissued - if so use token to get new secret_id
+                    if token != last_token:
+                        secret_id = retrieve_secret_id(
+                            url=vault_url,
+                            token=token
+                        )
+                        db.set('secret-id', secret_id)
+                        db.set('last-token', token)
+                        db.flush()
+
+                    ctxt = {
+                        'vault_url': vault_url,
+                        'role_id': json.loads(role_id),
+                        'secret_id': secret_id,
+                        'secret_backend': self.secret_backend,
+                    }
+                    vault_ca = data.get('vault_ca')
+                    if vault_ca:
+                        ctxt['vault_ca'] = json.loads(vault_ca)
+                    self.complete = True
+                    return ctxt
+        return {}
+
+
+def write_vaultlocker_conf(context, priority=100):
+    """Write vaultlocker configuration to disk and install alternative
+
+    :param context: Dict of data from vault-kv relation
+    :ptype: context: dict
+    :param priority: Priority of alternative configuration
+    :ptype: priority: int"""
+    charm_vl_path = "/var/lib/charm/{}/vaultlocker.conf".format(
+        hookenv.service_name()
+    )
+    host.mkdir(os.path.dirname(charm_vl_path), perms=0o700)
+    templating.render(source='vaultlocker.conf.j2',
+                      target=charm_vl_path,
+                      context=context, perms=0o600),
+    alternatives.install_alternative('vaultlocker.conf',
+                                     '/etc/vaultlocker/vaultlocker.conf',
+                                     charm_vl_path, priority)
+
+
+def vault_relation_complete(backend=None):
+    """Determine whether vault relation is complete
+
+    :param backend: Name of secrets backend requested
+    :ptype backend: string
+    :returns: whether the relation to vault is complete
+    :rtype: bool"""
+    vault_kv = VaultKVContext(secret_backend=backend or VAULTLOCKER_BACKEND)
+    vault_kv()
+    return vault_kv.complete
+
+
+# TODO: contrib a high level unwrap method to hvac that works
+def retrieve_secret_id(url, token):
+    """Retrieve a response-wrapped secret_id from Vault
+
+    :param url: URL to Vault Server
+    :ptype url: str
+    :param token: One shot Token to use
+    :ptype token: str
+    :returns: secret_id to use for Vault Access
+    :rtype: str"""
+    import hvac
+    client = hvac.Client(url=url, token=token)
+    response = client._post('/v1/sys/wrapping/unwrap')
+    if response.status_code == 200:
+        data = response.json()
+        return data['data']['secret_id']

charmhelpers/contrib/storage/linux/ceph.py

@@ -291,7 +291,7 @@ class Pool(object):
 
 class ReplicatedPool(Pool):
     def __init__(self, service, name, pg_num=None, replicas=2,
-                 percent_data=10.0):
+                 percent_data=10.0, app_name=None):
         super(ReplicatedPool, self).__init__(service=service, name=name)
         self.replicas = replicas
         if pg_num:
@@ -301,6 +301,10 @@ class ReplicatedPool(Pool):
             self.pg_num = min(pg_num, max_pgs)
         else:
             self.pg_num = self.get_pgs(self.replicas, percent_data)
+        if app_name:
+            self.app_name = app_name
+        else:
+            self.app_name = 'unknown'
 
     def create(self):
         if not pool_exists(self.service, self.name):
@@ -313,6 +317,12 @@ class ReplicatedPool(Pool):
                 update_pool(client=self.service,
                             pool=self.name,
                             settings={'size': str(self.replicas)})
+                try:
+                    set_app_name_for_pool(client=self.service,
+                                          pool=self.name,
+                                          name=self.app_name)
+                except CalledProcessError:
+                    log('Could not set app name for pool {}'.format(self.name, level=WARNING))
             except CalledProcessError:
                 raise
 
@@ -320,10 +330,14 @@ class ReplicatedPool(Pool):
 # Default jerasure erasure coded pool
 class ErasurePool(Pool):
     def __init__(self, service, name, erasure_code_profile="default",
-                 percent_data=10.0):
+                 percent_data=10.0, app_name=None):
         super(ErasurePool, self).__init__(service=service, name=name)
         self.erasure_code_profile = erasure_code_profile
         self.percent_data = percent_data
+        if app_name:
+            self.app_name = app_name
+        else:
+            self.app_name = 'unknown'
 
     def create(self):
         if not pool_exists(self.service, self.name):
@@ -355,6 +369,12 @@ class ErasurePool(Pool):
                    'erasure', self.erasure_code_profile]
             try:
                 check_call(cmd)
+                try:
+                    set_app_name_for_pool(client=self.service,
+                                          pool=self.name,
+                                          name=self.app_name)
+                except CalledProcessError:
+                    log('Could not set app name for pool {}'.format(self.name, level=WARNING))
             except CalledProcessError:
                 raise
 
@@ -778,6 +798,25 @@ def update_pool(client, pool, settings):
     check_call(cmd)
 
 
+def set_app_name_for_pool(client, pool, name):
+    """
+    Calls `osd pool application enable` for the specified pool name
+
+    :param client: Name of the ceph client to use
+    :type client: str
+    :param pool: Pool to set app name for
+    :type pool: str
+    :param name: app name for the specified pool
+    :type name: str
+
+    :raises: CalledProcessError if ceph call fails
+    """
+    if ceph_version() >= '12.0.0':
+        cmd = ['ceph', '--id', client, 'osd', 'pool',
+               'application', 'enable', pool, name]
+        check_call(cmd)
+
+
 def create_pool(service, name, replicas=3, pg_num=None):
     """Create a new RADOS pool."""
     if pool_exists(service, name):

charmhelpers/contrib/storage/linux/utils.py

@@ -67,3 +67,19 @@ def is_device_mounted(device):
     except Exception:
         return False
     return bool(re.search(r'MOUNTPOINT=".+"', out))
+
+
+def mkfs_xfs(device, force=False):
+    """Format device with XFS filesystem.
+
+    By default this should fail if the device already has a filesystem on it.
+    :param device: Full path to device to format
+    :ptype device: tr
+    :param force: Force operation
+    :ptype: force: boolean"""
+    cmd = ['mkfs.xfs']
+    if force:
+        cmd.append("-f")
+
+    cmd += ['-i', 'size=1024', device]
+    check_call(cmd)

charmhelpers/core/hookenv.py

@@ -290,7 +290,7 @@ class Config(dict):
         self.implicit_save = True
         self._prev_dict = None
         self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
-        if os.path.exists(self.path):
+        if os.path.exists(self.path) and os.stat(self.path).st_size:
             self.load_previous()
         atexit(self._implicit_save)
 
@@ -310,7 +310,11 @@ class Config(dict):
         """
         self.path = path or self.path
         with open(self.path) as f:
+            try:
                 self._prev_dict = json.load(f)
+            except ValueError as e:
+                log('Unable to parse previous config data - {}'.format(str(e)),
+                    level=ERROR)
         for k, v in copy.deepcopy(self._prev_dict).items():
             if k not in self:
                 self[k] = v
@@ -354,22 +358,40 @@ class Config(dict):
             self.save()
 
 
-@cached
+_cache_config = None
+
+
 def config(scope=None):
-    """Juju charm configuration"""
+    """
-    config_cmd_line = ['config-get']
+    Get the juju charm configuration (scope==None) or individual key,
-    if scope is not None:
+    (scope=str).  The returned value is a Python data structure loaded as
-        config_cmd_line.append(scope)
+    JSON from the Juju config command.
-    else:
+
-        config_cmd_line.append('--all')
+    :param scope: If set, return the value for the specified key.
-    config_cmd_line.append('--format=json')
+    :type scope: Optional[str]
+    :returns: Either the whole config as a Config, or a key from it.
+    :rtype: Any
+    """
+    global _cache_config
+    config_cmd_line = ['config-get', '--all', '--format=json']
     try:
+        # JSON Decode Exception for Python3.5+
+        exc_json = json.decoder.JSONDecodeError
+    except AttributeError:
+        # JSON Decode Exception for Python2.7 through Python3.4
+        exc_json = ValueError
+    try:
+        if _cache_config is None:
             config_data = json.loads(
                 subprocess.check_output(config_cmd_line).decode('UTF-8'))
+            _cache_config = Config(config_data)
         if scope is not None:
-            return config_data
+            return _cache_config.get(scope)
-        return Config(config_data)
+        return _cache_config
-    except ValueError:
+    except (exc_json, UnicodeDecodeError) as e:
+        log('Unable to parse output from config-get: config_cmd_line="{}" '
+            'message="{}"'
+            .format(config_cmd_line, str(e)), level=ERROR)
         return None
 
 
@@ -950,6 +972,13 @@ def application_version_set(version):
         log("Application Version: {}".format(version))
 
 
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def goal_state():
+    """Juju goal state values"""
+    cmd = ['goal-state', '--format=json']
+    return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)
 def is_leader():
     """Does the current unit hold the juju leadership

charmhelpers/core/host.py

@@ -972,6 +972,20 @@ def is_container():
 
 
 def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH):
+    """Adds the specified path to the mlocate's udpatedb.conf PRUNEPATH list.
+
+    This method has no effect if the path specified by updatedb_path does not
+    exist or is not a file.
+
+    @param path: string the path to add to the updatedb.conf PRUNEPATHS value
+    @param updatedb_path: the path the updatedb.conf file
+    """
+    if not os.path.exists(updatedb_path) or os.path.isdir(updatedb_path):
+        # If the updatedb.conf file doesn't exist then don't attempt to update
+        # the file as the package providing mlocate may not be installed on
+        # the local system
+        return
+
     with open(updatedb_path, 'r+') as f_id:
         updatedb_text = f_id.read()
         output = updatedb(updatedb_text, path)

charmhelpers/core/services/base.py

@@ -307,7 +307,9 @@ class PortManagerCallback(ManagerCallback):
     """
     def __call__(self, manager, service_name, event_name):
         service = manager.get_service(service_name)
-        new_ports = service.get('ports', [])
+        # turn this generator into a list,
+        # as we'll be going over it multiple times
+        new_ports = list(service.get('ports', []))
         port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name))
         if os.path.exists(port_file):
             with open(port_file) as fp:

charmhelpers/core/sysctl.py

@@ -31,18 +31,22 @@ __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
 def create(sysctl_dict, sysctl_file):
     """Creates a sysctl.conf file from a YAML associative array
 
-    :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
+    :param sysctl_dict: a dict or YAML-formatted string of sysctl
+                        options eg "{ 'kernel.max_pid': 1337 }"
     :type sysctl_dict: str
     :param sysctl_file: path to the sysctl file to be saved
     :type sysctl_file: str or unicode
     :returns: None
     """
+    if type(sysctl_dict) is not dict:
         try:
             sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
         except yaml.YAMLError:
             log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
                 level=ERROR)
             return
+    else:
+        sysctl_dict_parsed = sysctl_dict
 
     with open(sysctl_file, "w") as fd:
         for key, value in sysctl_dict_parsed.items():

charmhelpers/core/unitdata.py

@@ -166,6 +166,10 @@ class Storage(object):
 
     To support dicts, lists, integer, floats, and booleans values
     are automatically json encoded/decoded.
+
+    Note: to facilitate unit testing, ':memory:' can be passed as the
+    path parameter which causes sqlite3 to only build the db in memory.
+    This should only be used for testing purposes.
     """
     def __init__(self, path=None):
         self.db_path = path
@@ -175,6 +179,7 @@ class Storage(object):
             else:
                 self.db_path = os.path.join(
                     os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+        if self.db_path != ':memory:':
             with open(self.db_path, 'a') as f:
                 os.fchmod(f.fileno(), 0o600)
         self.conn = sqlite3.connect('%s' % self.db_path)

charmhelpers/fetch/ubuntu.py

@@ -44,6 +44,7 @@ ARCH_TO_PROPOSED_POCKET = {
     'x86_64': PROPOSED_POCKET,
     'ppc64le': PROPOSED_PORTS_POCKET,
     'aarch64': PROPOSED_PORTS_POCKET,
+    's390x': PROPOSED_PORTS_POCKET,
 }
 CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
 CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
@@ -157,6 +158,14 @@ CLOUD_ARCHIVE_POCKETS = {
     'queens/proposed': 'xenial-proposed/queens',
     'xenial-queens/proposed': 'xenial-proposed/queens',
     'xenial-proposed/queens': 'xenial-proposed/queens',
+    # Rocky
+    'rocky': 'bionic-updates/rocky',
+    'bionic-rocky': 'bionic-updates/rocky',
+    'bionic-rocky/updates': 'bionic-updates/rocky',
+    'bionic-updates/rocky': 'bionic-updates/rocky',
+    'rocky/proposed': 'bionic-proposed/rocky',
+    'bionic-rocky/proposed': 'bionic-proposed/rocky',
+    'bionic-proposed/rocky': 'bionic-proposed/rocky',
 }
 
 
@@ -306,7 +315,7 @@ def import_key(key):
         cmd = ['apt-key', 'adv', '--keyserver',
                'hkp://keyserver.ubuntu.com:80', '--recv-keys', key]
         try:
-            subprocess.check_call(cmd)
+            _run_with_retries(cmd)
         except subprocess.CalledProcessError:
             error = "Error importing PGP key '{}'".format(key)
             log(error)

tests/charmhelpers/contrib/amulet/deployment.py

@@ -50,7 +50,8 @@ class AmuletDeployment(object):
             this_service['units'] = 1
 
         self.d.add(this_service['name'], units=this_service['units'],
-                   constraints=this_service.get('constraints'))
+                   constraints=this_service.get('constraints'),
+                   storage=this_service.get('storage'))
 
         for svc in other_services:
             if 'location' in svc:
@@ -64,7 +65,8 @@ class AmuletDeployment(object):
                 svc['units'] = 1
 
             self.d.add(svc['name'], charm=branch_location, units=svc['units'],
-                       constraints=svc.get('constraints'))
+                       constraints=svc.get('constraints'),
+                       storage=svc.get('storage'))
 
     def _add_relations(self, relations):
         """Add all of the relations for the services."""

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -291,6 +291,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('zesty', None): self.zesty_ocata,
             ('artful', None): self.artful_pike,
             ('bionic', None): self.bionic_queens,
+            ('bionic', 'cloud:bionic-rocky'): self.bionic_rocky,
+            ('cosmic', None): self.cosmic_rocky,
         }
         return releases[(self.series, self.openstack)]
 
@@ -306,6 +308,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('zesty', 'ocata'),
             ('artful', 'pike'),
             ('bionic', 'queens'),
+            ('cosmic', 'rocky'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -40,6 +40,7 @@ import novaclient
 import pika
 import swiftclient
 
+from charmhelpers.core.decorators import retry_on_exception
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
@@ -55,7 +56,7 @@ OPENSTACK_RELEASES_PAIRS = [
     'trusty_mitaka', 'xenial_mitaka', 'xenial_newton',
     'yakkety_newton', 'xenial_ocata', 'zesty_ocata',
     'xenial_pike', 'artful_pike', 'xenial_queens',
-    'bionic_queens']
+    'bionic_queens', 'bionic_rocky', 'cosmic_rocky']
 
 
 class OpenStackAmuletUtils(AmuletUtils):
@@ -423,6 +424,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
+    @retry_on_exception(num_retries=5, base_delay=1)
     def keystone_wait_for_propagation(self, sentry_relation_pairs,
                                       api_version):
         """Iterate over list of sentry and relation tuples and verify that
@@ -542,7 +544,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         return ep
 
     def get_default_keystone_session(self, keystone_sentry,
-                                     openstack_release=None):
+                                     openstack_release=None, api_version=2):
         """Return a keystone session object and client object assuming standard
            default settings
 
@@ -557,12 +559,12 @@ class OpenStackAmuletUtils(AmuletUtils):
                eyc
         """
         self.log.debug('Authenticating keystone admin...')
-        api_version = 2
-        client_class = keystone_client.Client
         # 11 => xenial_queens
-        if openstack_release and openstack_release >= 11:
+        if api_version == 3 or (openstack_release and openstack_release >= 11):
-            api_version = 3
             client_class = keystone_client_v3.Client
+            api_version = 3
+        else:
+            client_class = keystone_client.Client
         keystone_ip = keystone_sentry.info['public-address']
         session, auth = self.get_keystone_session(
             keystone_ip,

tests/charmhelpers/core/hookenv.py

@@ -290,7 +290,7 @@ class Config(dict):
         self.implicit_save = True
         self._prev_dict = None
         self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
-        if os.path.exists(self.path):
+        if os.path.exists(self.path) and os.stat(self.path).st_size:
             self.load_previous()
         atexit(self._implicit_save)
 
@@ -310,7 +310,11 @@ class Config(dict):
         """
         self.path = path or self.path
         with open(self.path) as f:
+            try:
                 self._prev_dict = json.load(f)
+            except ValueError as e:
+                log('Unable to parse previous config data - {}'.format(str(e)),
+                    level=ERROR)
         for k, v in copy.deepcopy(self._prev_dict).items():
             if k not in self:
                 self[k] = v
@@ -354,22 +358,40 @@ class Config(dict):
             self.save()
 
 
-@cached
+_cache_config = None
+
+
 def config(scope=None):
-    """Juju charm configuration"""
+    """
-    config_cmd_line = ['config-get']
+    Get the juju charm configuration (scope==None) or individual key,
-    if scope is not None:
+    (scope=str).  The returned value is a Python data structure loaded as
-        config_cmd_line.append(scope)
+    JSON from the Juju config command.
-    else:
+
-        config_cmd_line.append('--all')
+    :param scope: If set, return the value for the specified key.
-    config_cmd_line.append('--format=json')
+    :type scope: Optional[str]
+    :returns: Either the whole config as a Config, or a key from it.
+    :rtype: Any
+    """
+    global _cache_config
+    config_cmd_line = ['config-get', '--all', '--format=json']
     try:
+        # JSON Decode Exception for Python3.5+
+        exc_json = json.decoder.JSONDecodeError
+    except AttributeError:
+        # JSON Decode Exception for Python2.7 through Python3.4
+        exc_json = ValueError
+    try:
+        if _cache_config is None:
             config_data = json.loads(
                 subprocess.check_output(config_cmd_line).decode('UTF-8'))
+            _cache_config = Config(config_data)
         if scope is not None:
-            return config_data
+            return _cache_config.get(scope)
-        return Config(config_data)
+        return _cache_config
-    except ValueError:
+    except (exc_json, UnicodeDecodeError) as e:
+        log('Unable to parse output from config-get: config_cmd_line="{}" '
+            'message="{}"'
+            .format(config_cmd_line, str(e)), level=ERROR)
         return None
 
 
@@ -950,6 +972,13 @@ def application_version_set(version):
         log("Application Version: {}".format(version))
 
 
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def goal_state():
+    """Juju goal state values"""
+    cmd = ['goal-state', '--format=json']
+    return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)
 def is_leader():
     """Does the current unit hold the juju leadership

tests/charmhelpers/core/host.py

@@ -972,6 +972,20 @@ def is_container():
 
 
 def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH):
+    """Adds the specified path to the mlocate's udpatedb.conf PRUNEPATH list.
+
+    This method has no effect if the path specified by updatedb_path does not
+    exist or is not a file.
+
+    @param path: string the path to add to the updatedb.conf PRUNEPATHS value
+    @param updatedb_path: the path the updatedb.conf file
+    """
+    if not os.path.exists(updatedb_path) or os.path.isdir(updatedb_path):
+        # If the updatedb.conf file doesn't exist then don't attempt to update
+        # the file as the package providing mlocate may not be installed on
+        # the local system
+        return
+
     with open(updatedb_path, 'r+') as f_id:
         updatedb_text = f_id.read()
         output = updatedb(updatedb_text, path)

tests/charmhelpers/core/services/base.py

@@ -307,7 +307,9 @@ class PortManagerCallback(ManagerCallback):
     """
     def __call__(self, manager, service_name, event_name):
         service = manager.get_service(service_name)
-        new_ports = service.get('ports', [])
+        # turn this generator into a list,
+        # as we'll be going over it multiple times
+        new_ports = list(service.get('ports', []))
         port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name))
         if os.path.exists(port_file):
             with open(port_file) as fp:

tests/charmhelpers/core/sysctl.py

@@ -31,18 +31,22 @@ __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
 def create(sysctl_dict, sysctl_file):
     """Creates a sysctl.conf file from a YAML associative array
 
-    :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
+    :param sysctl_dict: a dict or YAML-formatted string of sysctl
+                        options eg "{ 'kernel.max_pid': 1337 }"
     :type sysctl_dict: str
     :param sysctl_file: path to the sysctl file to be saved
     :type sysctl_file: str or unicode
     :returns: None
     """
+    if type(sysctl_dict) is not dict:
         try:
             sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
         except yaml.YAMLError:
             log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
                 level=ERROR)
             return
+    else:
+        sysctl_dict_parsed = sysctl_dict
 
     with open(sysctl_file, "w") as fd:
         for key, value in sysctl_dict_parsed.items():

tests/charmhelpers/core/unitdata.py

@@ -166,6 +166,10 @@ class Storage(object):
 
     To support dicts, lists, integer, floats, and booleans values
     are automatically json encoded/decoded.
+
+    Note: to facilitate unit testing, ':memory:' can be passed as the
+    path parameter which causes sqlite3 to only build the db in memory.
+    This should only be used for testing purposes.
     """
     def __init__(self, path=None):
         self.db_path = path
@@ -175,6 +179,7 @@ class Storage(object):
             else:
                 self.db_path = os.path.join(
                     os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+        if self.db_path != ':memory:':
             with open(self.db_path, 'a') as f:
                 os.fchmod(f.fileno(), 0o600)
         self.conn = sqlite3.connect('%s' % self.db_path)