Sync charm-helpers

Notable issues resolved:

openstack_upgrade_available() broken for swift
https://bugs.launchpad.net/charm-swift-proxy/+bug/1743847

haproxy context doesn't consider bindings
https://bugs.launchpad.net/charm-helpers/+bug/1735421

regression in haproxy check
https://bugs.launchpad.net/charm-helpers/+bug/1743287

Change-Id: I92b1f8c534dc5afc6ac3335c9eb3571f38784fe1

hooks/charmhelpers/contrib/openstack/context.py

@@ -93,10 +93,10 @@ from charmhelpers.contrib.network.ip import (
     format_ipv6_addr,
     is_bridge_member,
     is_ipv6_disabled,
+    get_relation_ip,
 )
 from charmhelpers.contrib.openstack.utils import (
     config_flags_parser,
-    get_host_ip,
     enable_memcache,
     snap_install_requested,
     CompareOpenStackReleases,
@@ -626,11 +626,6 @@ class HAProxyContext(OSContextGenerator):
         if not relation_ids('cluster') and not self.singlenode_mode:
             return {}
 
-        if config('prefer-ipv6'):
-            addr = get_ipv6_addr(exc_list=[config('vip')])[0]
-        else:
-            addr = get_host_ip(unit_get('private-address'))
-
         l_unit = local_unit().replace('/', '-')
         cluster_hosts = {}
 
@@ -638,7 +633,15 @@ class HAProxyContext(OSContextGenerator):
         # and associated backends
         for addr_type in ADDRESS_TYPES:
             cfg_opt = 'os-{}-network'.format(addr_type)
-            laddr = get_address_in_network(config(cfg_opt))
+            # NOTE(thedac) For some reason the ADDRESS_MAP uses 'int' rather
+            # than 'internal'
+            if addr_type == 'internal':
+                _addr_map_type = INTERNAL
+            else:
+                _addr_map_type = addr_type
+            # Network spaces aware
+            laddr = get_relation_ip(ADDRESS_MAP[_addr_map_type]['binding'],
+                                    config(cfg_opt))
             if laddr:
                 netmask = get_netmask_for_address(laddr)
                 cluster_hosts[laddr] = {
@@ -649,15 +652,19 @@ class HAProxyContext(OSContextGenerator):
                 }
                 for rid in relation_ids('cluster'):
                     for unit in sorted(related_units(rid)):
+                        # API Charms will need to set {addr_type}-address with
+                        # get_relation_ip(addr_type)
                         _laddr = relation_get('{}-address'.format(addr_type),
                                               rid=rid, unit=unit)
                         if _laddr:
                             _unit = unit.replace('/', '-')
                             cluster_hosts[laddr]['backends'][_unit] = _laddr
 
-        # NOTE(jamespage) add backend based on private address - this
+        # NOTE(jamespage) add backend based on get_relation_ip - this
-        # with either be the only backend or the fallback if no acls
+        # will either be the only backend or the fallback if no acls
         # match in the frontend
+        # Network spaces aware
+        addr = get_relation_ip('cluster')
         cluster_hosts[addr] = {}
         netmask = get_netmask_for_address(addr)
         cluster_hosts[addr] = {
@@ -667,6 +674,8 @@ class HAProxyContext(OSContextGenerator):
         }
         for rid in relation_ids('cluster'):
             for unit in sorted(related_units(rid)):
+                # API Charms will need to set their private-address with
+                # get_relation_ip('cluster')
                 _laddr = relation_get('private-address',
                                       rid=rid, unit=unit)
                 if _laddr:

hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh

@@ -9,7 +9,7 @@
 CRITICAL=0
 NOTACTIVE=''
 LOGFILE=/var/log/nagios/check_haproxy.log
-AUTH=$(grep -r "stats auth" /etc/haproxy/haproxy.cfg | awk 'NR=1{print $4}')
+AUTH=$(grep -r "stats auth" /etc/haproxy/haproxy.cfg | awk 'NR=1{print $3}')
 
 typeset -i N_INSTANCES=0
 for appserver in $(awk '/^\s+server/{print $2}' /etc/haproxy/haproxy.cfg)

hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh

@@ -10,7 +10,7 @@
 CURRQthrsh=0
 MAXQthrsh=100
 
-AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
+AUTH=$(grep -r "stats auth" /etc/haproxy/haproxy.cfg | awk 'NR=1{print $3}')
 
 HAPROXYSTATS=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v)
 

hooks/charmhelpers/contrib/openstack/ha/utils.py

@@ -23,6 +23,8 @@
 Helpers for high availability.
 """
 
+import json
+
 import re
 
 from charmhelpers.core.hookenv import (
@@ -32,6 +34,7 @@ from charmhelpers.core.hookenv import (
     config,
     status_set,
     DEBUG,
+    WARNING,
 )
 
 from charmhelpers.core.host import (
@@ -40,6 +43,23 @@ from charmhelpers.core.host import (
 
 from charmhelpers.contrib.openstack.ip import (
     resolve_address,
+    is_ipv6,
+)
+
+from charmhelpers.contrib.network.ip import (
+    get_iface_for_address,
+    get_netmask_for_address,
+)
+
+from charmhelpers.contrib.hahelpers.cluster import (
+    get_hacluster_config
+)
+
+JSON_ENCODE_OPTIONS = dict(
+    sort_keys=True,
+    allow_nan=False,
+    indent=None,
+    separators=(',', ':'),
 )
 
 
@@ -53,8 +73,8 @@ class DNSHAException(Exception):
 def update_dns_ha_resource_params(resources, resource_params,
                                   relation_id=None,
                                   crm_ocf='ocf:maas:dns'):
-    """ Check for os-*-hostname settings and update resource dictionaries for
+    """ Configure DNS-HA resources based on provided configuration and
-    the HA relation.
+    update resource dictionaries for the HA relation.
 
     @param resources: Pointer to dictionary of resources.
                       Usually instantiated in ha_joined().
@@ -64,7 +84,85 @@ def update_dns_ha_resource_params(resources, resource_params,
     @param crm_ocf: Corosync Open Cluster Framework resource agent to use for
                     DNS HA
     """
+    _relation_data = {'resources': {}, 'resource_params': {}}
+    update_hacluster_dns_ha(charm_name(),
+                            _relation_data,
+                            crm_ocf)
+    resources.update(_relation_data['resources'])
+    resource_params.update(_relation_data['resource_params'])
+    relation_set(relation_id=relation_id, groups=_relation_data['groups'])
 
+
+def assert_charm_supports_dns_ha():
+    """Validate prerequisites for DNS HA
+    The MAAS client is only available on Xenial or greater
+
+    :raises DNSHAException: if release is < 16.04
+    """
+    if lsb_release().get('DISTRIB_RELEASE') < '16.04':
+        msg = ('DNS HA is only supported on 16.04 and greater '
+               'versions of Ubuntu.')
+        status_set('blocked', msg)
+        raise DNSHAException(msg)
+    return True
+
+
+def expect_ha():
+    """ Determine if the unit expects to be in HA
+
+    Check for VIP or dns-ha settings which indicate the unit should expect to
+    be related to hacluster.
+
+    @returns boolean
+    """
+    return config('vip') or config('dns-ha')
+
+
+def generate_ha_relation_data(service):
+    """ Generate relation data for ha relation
+
+    Based on configuration options and unit interfaces, generate a json
+    encoded dict of relation data items for the hacluster relation,
+    providing configuration for DNS HA or VIP's + haproxy clone sets.
+
+    @returns dict: json encoded data for use with relation_set
+    """
+    _haproxy_res = 'res_{}_haproxy'.format(service)
+    _relation_data = {
+        'resources': {
+            _haproxy_res: 'lsb:haproxy',
+        },
+        'resource_params': {
+            _haproxy_res: 'op monitor interval="5s"'
+        },
+        'init_services': {
+            _haproxy_res: 'haproxy'
+        },
+        'clones': {
+            'cl_{}_haproxy'.format(service): _haproxy_res
+        },
+    }
+
+    if config('dns-ha'):
+        update_hacluster_dns_ha(service, _relation_data)
+    else:
+        update_hacluster_vip(service, _relation_data)
+
+    return {
+        'json_{}'.format(k): json.dumps(v, **JSON_ENCODE_OPTIONS)
+        for k, v in _relation_data.items() if v
+    }
+
+
+def update_hacluster_dns_ha(service, relation_data,
+                            crm_ocf='ocf:maas:dns'):
+    """ Configure DNS-HA resources based on provided configuration
+
+    @param service: Name of the service being configured
+    @param relation_data: Pointer to dictionary of relation data.
+    @param crm_ocf: Corosync Open Cluster Framework resource agent to use for
+                    DNS HA
+    """
     # Validate the charm environment for DNS HA
     assert_charm_supports_dns_ha()
 
@@ -93,7 +191,7 @@ def update_dns_ha_resource_params(resources, resource_params,
             status_set('blocked', msg)
             raise DNSHAException(msg)
 
-        hostname_key = 'res_{}_{}_hostname'.format(charm_name(), endpoint_type)
+        hostname_key = 'res_{}_{}_hostname'.format(service, endpoint_type)
         if hostname_key in hostname_group:
             log('DNS HA: Resource {}: {} already exists in '
                 'hostname group - skipping'.format(hostname_key, hostname),
@@ -101,42 +199,67 @@ def update_dns_ha_resource_params(resources, resource_params,
             continue
 
         hostname_group.append(hostname_key)
-        resources[hostname_key] = crm_ocf
+        relation_data['resources'][hostname_key] = crm_ocf
-        resource_params[hostname_key] = (
+        relation_data['resource_params'][hostname_key] = (
-            'params fqdn="{}" ip_address="{}" '
+            'params fqdn="{}" ip_address="{}"'
-            ''.format(hostname, resolve_address(endpoint_type=endpoint_type,
+            .format(hostname, resolve_address(endpoint_type=endpoint_type,
-                                                override=False)))
+                                              override=False)))
 
     if len(hostname_group) >= 1:
         log('DNS HA: Hostname group is set with {} as members. '
             'Informing the ha relation'.format(' '.join(hostname_group)),
             DEBUG)
-        relation_set(relation_id=relation_id, groups={
+        relation_data['groups'] = {
-            'grp_{}_hostnames'.format(charm_name()): ' '.join(hostname_group)})
+            'grp_{}_hostnames'.format(service): ' '.join(hostname_group)
+        }
     else:
         msg = 'DNS HA: Hostname group has no members.'
         status_set('blocked', msg)
         raise DNSHAException(msg)
 
 
-def assert_charm_supports_dns_ha():
+def update_hacluster_vip(service, relation_data):
-    """Validate prerequisites for DNS HA
+    """ Configure VIP resources based on provided configuration
-    The MAAS client is only available on Xenial or greater
+
+    @param service: Name of the service being configured
+    @param relation_data: Pointer to dictionary of relation data.
     """
-    if lsb_release().get('DISTRIB_RELEASE') < '16.04':
+    cluster_config = get_hacluster_config()
-        msg = ('DNS HA is only supported on 16.04 and greater '
+    vip_group = []
-               'versions of Ubuntu.')
+    for vip in cluster_config['vip'].split():
-        status_set('blocked', msg)
+        if is_ipv6(vip):
-        raise DNSHAException(msg)
+            res_neutron_vip = 'ocf:heartbeat:IPv6addr'
-    return True
+            vip_params = 'ipv6addr'
+        else:
+            res_neutron_vip = 'ocf:heartbeat:IPaddr2'
+            vip_params = 'ip'
 
+        iface = (get_iface_for_address(vip) or
+                 config('vip_iface'))
+        netmask = (get_netmask_for_address(vip) or
+                   config('vip_cidr'))
 
-def expect_ha():
+        if iface is not None:
-    """ Determine if the unit expects to be in HA
+            vip_key = 'res_{}_{}_vip'.format(service, iface)
+            if vip_key in vip_group:
+                if vip not in relation_data['resource_params'][vip_key]:
+                    vip_key = '{}_{}'.format(vip_key, vip_params)
+                else:
+                    log("Resource '%s' (vip='%s') already exists in "
+                        "vip group - skipping" % (vip_key, vip), WARNING)
+                    continue
 
-    Check for VIP or dns-ha settings which indicate the unit should expect to
+            relation_data['resources'][vip_key] = res_neutron_vip
-    be related to hacluster.
+            relation_data['resource_params'][vip_key] = (
+                'params {ip}="{vip}" cidr_netmask="{netmask}" '
+                'nic="{iface}"'.format(ip=vip_params,
+                                       vip=vip,
+                                       iface=iface,
+                                       netmask=netmask)
+            )
+            vip_group.append(vip_key)
 
-    @returns boolean
+    if len(vip_group) >= 1:
-    """
+        relation_data['groups'] = {
-    return config('vip') or config('dns-ha')
+            'grp_{}_vips'.format(service): ' '.join(vip_group)
+        }

hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf

@@ -15,9 +15,6 @@ Listen {{ public_port }}
 {% if port -%}
 <VirtualHost *:{{ port }}>
     WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
-{% if python_path -%}
-                      python-path={{ python_path }} \
-{% endif -%}
                       display-name=%{GROUP}
     WSGIProcessGroup {{ service_name }}
     WSGIScriptAlias / {{ script }}
@@ -29,7 +26,7 @@ Listen {{ public_port }}
     ErrorLog /var/log/apache2/{{ service_name }}_error.log
     CustomLog /var/log/apache2/{{ service_name }}_access.log combined
 
-    <Directory {{ usr_bin }}>
+    <Directory /usr/bin>
         <IfVersion >= 2.4>
             Require all granted
         </IfVersion>
@@ -44,9 +41,6 @@ Listen {{ public_port }}
 {% if admin_port -%}
 <VirtualHost *:{{ admin_port }}>
     WSGIDaemonProcess {{ service_name }}-admin processes={{ admin_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
-{% if python_path -%}
-                      python-path={{ python_path }} \
-{% endif -%}
                       display-name=%{GROUP}
     WSGIProcessGroup {{ service_name }}-admin
     WSGIScriptAlias / {{ admin_script }}
@@ -58,7 +52,7 @@ Listen {{ public_port }}
     ErrorLog /var/log/apache2/{{ service_name }}_error.log
     CustomLog /var/log/apache2/{{ service_name }}_access.log combined
 
-    <Directory {{ usr_bin }}>
+    <Directory /usr/bin>
         <IfVersion >= 2.4>
             Require all granted
         </IfVersion>
@@ -73,9 +67,6 @@ Listen {{ public_port }}
 {% if public_port -%}
 <VirtualHost *:{{ public_port }}>
     WSGIDaemonProcess {{ service_name }}-public processes={{ public_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
-{% if python_path -%}
-                      python-path={{ python_path }} \
-{% endif -%}
                       display-name=%{GROUP}
     WSGIProcessGroup {{ service_name }}-public
     WSGIScriptAlias / {{ public_script }}
@@ -87,7 +78,7 @@ Listen {{ public_port }}
     ErrorLog /var/log/apache2/{{ service_name }}_error.log
     CustomLog /var/log/apache2/{{ service_name }}_access.log combined
 
-    <Directory {{ usr_bin }}>
+    <Directory /usr/bin>
         <IfVersion >= 2.4>
             Require all granted
         </IfVersion>

hooks/charmhelpers/contrib/openstack/utils.py

@@ -626,11 +626,6 @@ def openstack_upgrade_available(package):
     else:
         avail_vers = get_os_version_install_source(src)
     apt.init()
-    if "swift" in package:
-        major_cur_vers = cur_vers.split('.', 1)[0]
-        major_avail_vers = avail_vers.split('.', 1)[0]
-        major_diff = apt.version_compare(major_avail_vers, major_cur_vers)
-        return avail_vers > cur_vers and (major_diff == 1 or major_diff == 0)
     return apt.version_compare(avail_vers, cur_vers) == 1
 
 

hooks/charmhelpers/core/hookenv.py

@@ -39,6 +39,7 @@ if not six.PY3:
 else:
     from collections import UserDict
 
+
 CRITICAL = "CRITICAL"
 ERROR = "ERROR"
 WARNING = "WARNING"
@@ -344,6 +345,7 @@ class Config(dict):
 
         """
         with open(self.path, 'w') as f:
+            os.fchmod(f.fileno(), 0o600)
             json.dump(self, f)
 
     def _implicit_save(self):

hooks/charmhelpers/core/unitdata.py

@@ -175,6 +175,8 @@ class Storage(object):
             else:
                 self.db_path = os.path.join(
                     os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+        with open(self.db_path, 'a') as f:
+            os.fchmod(f.fileno(), 0o600)
         self.conn = sqlite3.connect('%s' % self.db_path)
         self.cursor = self.conn.cursor()
         self.revision = None

tests/charmhelpers/contrib/openstack/utils.py

@@ -626,11 +626,6 @@ def openstack_upgrade_available(package):
     else:
         avail_vers = get_os_version_install_source(src)
     apt.init()
-    if "swift" in package:
-        major_cur_vers = cur_vers.split('.', 1)[0]
-        major_avail_vers = avail_vers.split('.', 1)[0]
-        major_diff = apt.version_compare(major_avail_vers, major_cur_vers)
-        return avail_vers > cur_vers and (major_diff == 1 or major_diff == 0)
     return apt.version_compare(avail_vers, cur_vers) == 1
 
 

tests/charmhelpers/core/hookenv.py

@@ -39,6 +39,7 @@ if not six.PY3:
 else:
     from collections import UserDict
 
+
 CRITICAL = "CRITICAL"
 ERROR = "ERROR"
 WARNING = "WARNING"
@@ -344,6 +345,7 @@ class Config(dict):
 
         """
         with open(self.path, 'w') as f:
+            os.fchmod(f.fileno(), 0o600)
             json.dump(self, f)
 
     def _implicit_save(self):

tests/charmhelpers/core/unitdata.py

@@ -175,6 +175,8 @@ class Storage(object):
             else:
                 self.db_path = os.path.join(
                     os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+        with open(self.db_path, 'a') as f:
+            os.fchmod(f.fileno(), 0o600)
         self.conn = sqlite3.connect('%s' % self.db_path)
         self.cursor = self.conn.cursor()
         self.revision = None