Sync charm-helpers

Change-Id: I7436d0d90b90fb1b6f97259387d11d879eee3a1c

charmhelpers/contrib/charmsupport/nrpe.py

@@ -33,6 +33,7 @@ from charmhelpers.core.hookenv import (
     hook_name,
     local_unit,
     log,
+    relation_get,
     relation_ids,
     relation_set,
     relations_of_type,
@@ -260,11 +261,23 @@ class NRPE(object):
         relation = relation_ids('nrpe-external-master')
         if relation:
             log("Setting charm primary status {}".format(primary))
-            for rid in relation_ids('nrpe-external-master'):
+            for rid in relation:
                 relation_set(relation_id=rid, relation_settings={'primary': self.primary})
+        self.remove_check_queue = set()
 
     def add_check(self, *args, **kwargs):
+        shortname = None
+        if kwargs.get('shortname') is None:
+            if len(args) > 0:
+                shortname = args[0]
+        else:
+            shortname = kwargs['shortname']
+
         self.checks.append(Check(*args, **kwargs))
+        try:
+            self.remove_check_queue.remove(shortname)
+        except KeyError:
+            pass
 
     def remove_check(self, *args, **kwargs):
         if kwargs.get('shortname') is None:
@@ -281,6 +294,7 @@ class NRPE(object):
 
         check = Check(*args, **kwargs)
         check.remove(self.hostname)
+        self.remove_check_queue.add(kwargs['shortname'])
 
     def write(self):
         try:
@@ -313,7 +327,24 @@ class NRPE(object):
         monitor_ids = relation_ids("local-monitors") + \
             relation_ids("nrpe-external-master")
         for rid in monitor_ids:
-            relation_set(relation_id=rid, monitors=yaml.dump(monitors))
+            reldata = relation_get(unit=local_unit(), rid=rid)
+            if 'monitors' in reldata:
+                # update the existing set of monitors with the new data
+                old_monitors = yaml.safe_load(reldata['monitors'])
+                old_nrpe_monitors = old_monitors['monitors']['remote']['nrpe']
+                # remove keys that are in the remove_check_queue
+                old_nrpe_monitors = {k: v for k, v in old_nrpe_monitors.items()
+                                     if k not in self.remove_check_queue}
+                # update/add nrpe_monitors
+                old_nrpe_monitors.update(nrpe_monitors)
+                old_monitors['monitors']['remote']['nrpe'] = old_nrpe_monitors
+                # write back to the relation
+                relation_set(relation_id=rid, monitors=yaml.dump(old_monitors))
+            else:
+                # write a brand new set of monitors, as no existing ones.
+                relation_set(relation_id=rid, monitors=yaml.dump(monitors))
+
+        self.remove_check_queue.clear()
 
 
 def get_nagios_hostcontext(relation_name='nrpe-external-master'):

charmhelpers/contrib/openstack/amulet/deployment.py

@@ -323,6 +323,23 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         else:
             return releases[self.series]
 
+    def get_percona_service_entry(self, memory_constraint=None):
+        """Return a amulet service entry for percona cluster.
+
+        :param memory_constraint: Override the default memory constraint
+                                  in the service entry.
+        :type memory_constraint: str
+        :returns: Amulet service entry.
+        :rtype: dict
+        """
+        memory_constraint = memory_constraint or '3072M'
+        svc_entry = {
+            'name': 'percona-cluster',
+            'constraints': {'mem': memory_constraint}}
+        if self._get_openstack_release() <= self.trusty_mitaka:
+            svc_entry['location'] = 'cs:trusty/percona-cluster'
+        return svc_entry
+
     def get_ceph_expected_pools(self, radosgw=False):
         """Return a list of expected ceph pools in a ceph + cinder + glance
         test scenario, based on OpenStack release and whether ceph radosgw

charmhelpers/contrib/openstack/audits/openstack_security_guide.py

@@ -126,7 +126,11 @@ def _config_ini(path):
     :returns: Configuration contained in path
     :rtype: Dict
     """
-    conf = configparser.ConfigParser()
+    # When strict is enabled, duplicate options are not allowed in the
+    # parsed INI; however, Oslo allows duplicate values. This change
+    # causes us to ignore the duplicate values which is acceptable as
+    # long as we don't validate any multi-value options
+    conf = configparser.ConfigParser(strict=False)
     conf.read(path)
     return dict(conf)
 
@@ -204,7 +208,7 @@ def validate_file_ownership(config):
                     "Invalid ownership configuration: {}".format(key))
         owner = options.get('owner', config.get('owner', 'root'))
         group = options.get('group', config.get('group', 'root'))
-        optional = options.get('optional', config.get('optional', 'False'))
+        optional = options.get('optional', config.get('optional', False))
         if '*' in file_name:
             for file in glob.glob(file_name):
                 if file not in files.keys():
@@ -226,7 +230,7 @@ def validate_file_permissions(config):
                 raise RuntimeError(
                     "Invalid ownership configuration: {}".format(key))
         mode = options.get('mode', config.get('permissions', '600'))
-        optional = options.get('optional', config.get('optional', 'False'))
+        optional = options.get('optional', config.get('optional', False))
         if '*' in file_name:
             for file in glob.glob(file_name):
                 if file not in files.keys():

charmhelpers/contrib/openstack/cert_utils.py

@@ -106,9 +106,11 @@ class CertRequest(object):
             sans = sorted(list(set(entry['addresses'])))
             request[entry['cn']] = {'sans': sans}
         if self.json_encode:
-            return {'cert_requests': json.dumps(request, sort_keys=True)}
+            req = {'cert_requests': json.dumps(request, sort_keys=True)}
         else:
-            return {'cert_requests': request}
+            req = {'cert_requests': request}
+        req['unit_name'] = local_unit().replace('/', '_')
+        return req
 
 
 def get_certificate_request(json_encode=True):

charmhelpers/contrib/openstack/context.py

@@ -258,7 +258,7 @@ class SharedDBContext(OSContextGenerator):
                     'database_password': rdata.get(password_setting),
                     'database_type': 'mysql+pymysql'
                 }
-                if CompareOpenStackReleases(rel) < 'stein':
+                if CompareOpenStackReleases(rel) < 'queens':
                     ctxt['database_type'] = 'mysql'
                 if self.context_complete(ctxt):
                     db_ssl(rdata, ctxt, self.ssl_dir)
@@ -443,8 +443,10 @@ class IdentityServiceContext(OSContextGenerator):
                              'api_version': api_version})
 
                 if float(api_version) > 2:
-                    ctxt.update({'admin_domain_name':
-                                 rdata.get('service_domain')})
+                    ctxt.update({
+                        'admin_domain_name': rdata.get('service_domain'),
+                        'service_project_id': rdata.get('service_tenant_id'),
+                        'service_domain_id': rdata.get('service_domain_id')})
 
                 # we keep all veriables in ctxt for compatibility and
                 # add nested dictionary for keystone_authtoken generic
@@ -1710,6 +1712,10 @@ class NeutronAPIContext(OSContextGenerator):
                 'rel_key': 'enable-nsg-logging',
                 'default': False,
             },
+            'enable_nfg_logging': {
+                'rel_key': 'enable-nfg-logging',
+                'default': False,
+            },
             'global_physnet_mtu': {
                 'rel_key': 'global-physnet-mtu',
                 'default': 1500,

charmhelpers/contrib/storage/linux/ceph.py

@@ -1482,6 +1482,21 @@ def send_request_if_needed(request, relation='ceph'):
             relation_set(relation_id=rid, broker_req=request.request)
 
 
+def has_broker_rsp(rid=None, unit=None):
+    """Return True if the broker_rsp key is 'truthy' (i.e. set to something) in the relation data.
+
+    :param rid: The relation to check (default of None means current relation)
+    :type rid: Union[str, None]
+    :param unit: The remote unit to check (default of None means current unit)
+    :type unit: Union[str, None]
+    :returns: True if broker key exists and is set to something 'truthy'
+    :rtype: bool
+    """
+    rdata = relation_get(rid=rid, unit=unit) or {}
+    broker_rsp = rdata.get(get_broker_rsp_key())
+    return True if broker_rsp else False
+
+
 def is_broker_action_done(action, rid=None, unit=None):
     """Check whether broker action has completed yet.
 

charmhelpers/contrib/storage/linux/utils.py

@@ -110,17 +110,19 @@ def is_device_mounted(device):
     return bool(re.search(r'MOUNTPOINT=".+"', out))
 
 
-def mkfs_xfs(device, force=False):
+def mkfs_xfs(device, force=False, inode_size=1024):
     """Format device with XFS filesystem.
 
     By default this should fail if the device already has a filesystem on it.
     :param device: Full path to device to format
     :ptype device: tr
     :param force: Force operation
-    :ptype: force: boolean"""
+    :ptype: force: boolean
+    :param inode_size: XFS inode size in bytes
+    :ptype inode_size: int"""
     cmd = ['mkfs.xfs']
     if force:
         cmd.append("-f")
 
-    cmd += ['-i', 'size=1024', device]
+    cmd += ['-i', "size={}".format(inode_size), device]
     check_call(cmd)

tests/basic_deployment.py

@@ -100,7 +100,7 @@ class NovaCCBasicDeployment(OpenStackAmuletDeployment):
             {'name': 'nova-compute', 'units': 2},
             {'name': 'keystone'},
             {'name': 'glance'},
-            {'name': 'percona-cluster', 'constraints': {'mem': '3072M'}},
+            self.get_percona_service_entry(),
         ]
         if self._get_openstack_release() >= self.xenial_ocata:
             other_ocata_services = [

tox.ini

@@ -94,7 +94,7 @@ basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
-    bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-rocky --no-destroy
+    bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-stein --no-destroy
 
 [testenv:func27-dfs]
 # Charm Functional Test