Sync charm-helpers

Notable issues resolved: openstack_upgrade_available() broken for swift https://bugs.launchpad.net/charm-swift-proxy/+bug/1743847 haproxy context doesn't consider bindings https://bugs.launchpad.net/charm-helpers/+bug/1735421 regression in haproxy check https://bugs.launchpad.net/charm-helpers/+bug/1743287 Change-Id: I4e53ae77afb5b0afa5219941ddfddc87e2e601d6
2018-01-19 12:09:44 +00:00 · 2018-01-19 12:09:44 +00:00 · 208e5f950a
commit 208e5f950a
parent fe9633856b
10 changed files with 180 additions and 54 deletions
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@ -93,10 +93,10 @@ from charmhelpers.contrib.network.ip import (
    format_ipv6_addr,
    is_bridge_member,
    is_ipv6_disabled,
+    get_relation_ip,
 )
 from charmhelpers.contrib.openstack.utils import (
    config_flags_parser,
-    get_host_ip,
    enable_memcache,
    snap_install_requested,
    CompareOpenStackReleases,
@ -626,11 +626,6 @@ class HAProxyContext(OSContextGenerator):
        if not relation_ids('cluster') and not self.singlenode_mode:
            return {}

-        if config('prefer-ipv6'):
-            addr = get_ipv6_addr(exc_list=[config('vip')])[0]
-        else:
-            addr = get_host_ip(unit_get('private-address'))
-
        l_unit = local_unit().replace('/', '-')
        cluster_hosts = {}

@ -638,7 +633,15 @@ class HAProxyContext(OSContextGenerator):
        # and associated backends
        for addr_type in ADDRESS_TYPES:
            cfg_opt = 'os-{}-network'.format(addr_type)
-            laddr = get_address_in_network(config(cfg_opt))
+            # NOTE(thedac) For some reason the ADDRESS_MAP uses 'int' rather
+            # than 'internal'
+            if addr_type == 'internal':
+                _addr_map_type = INTERNAL
+            else:
+                _addr_map_type = addr_type
+            # Network spaces aware
+            laddr = get_relation_ip(ADDRESS_MAP[_addr_map_type]['binding'],
+                                    config(cfg_opt))
            if laddr:
                netmask = get_netmask_for_address(laddr)
                cluster_hosts[laddr] = {
@ -649,15 +652,19 @@ class HAProxyContext(OSContextGenerator):
                }
                for rid in relation_ids('cluster'):
                    for unit in sorted(related_units(rid)):
+                        # API Charms will need to set {addr_type}-address with
+                        # get_relation_ip(addr_type)
                        _laddr = relation_get('{}-address'.format(addr_type),
                                              rid=rid, unit=unit)
                        if _laddr:
                            _unit = unit.replace('/', '-')
                            cluster_hosts[laddr]['backends'][_unit] = _laddr

-        # NOTE(jamespage) add backend based on private address - this
-        # with either be the only backend or the fallback if no acls
+        # NOTE(jamespage) add backend based on get_relation_ip - this
+        # will either be the only backend or the fallback if no acls
        # match in the frontend
+        # Network spaces aware
+        addr = get_relation_ip('cluster')
        cluster_hosts[addr] = {}
        netmask = get_netmask_for_address(addr)
        cluster_hosts[addr] = {
@ -667,6 +674,8 @@ class HAProxyContext(OSContextGenerator):
        }
        for rid in relation_ids('cluster'):
            for unit in sorted(related_units(rid)):
+                # API Charms will need to set their private-address with
+                # get_relation_ip('cluster')
                _laddr = relation_get('private-address',
                                      rid=rid, unit=unit)
                if _laddr:
--- a/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh
+++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh
@ -9,7 +9,7 @@
 CRITICAL=0
 NOTACTIVE=''
 LOGFILE=/var/log/nagios/check_haproxy.log
-AUTH=$(grep -r "stats auth" /etc/haproxy/haproxy.cfg | awk 'NR=1{print $4}')
+AUTH=$(grep -r "stats auth" /etc/haproxy/haproxy.cfg | awk 'NR=1{print $3}')

 typeset -i N_INSTANCES=0
 for appserver in $(awk '/^\s+server/{print $2}' /etc/haproxy/haproxy.cfg)
--- a/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh
+++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh
@ -10,7 +10,7 @@
 CURRQthrsh=0
 MAXQthrsh=100

-AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
+AUTH=$(grep -r "stats auth" /etc/haproxy/haproxy.cfg | awk 'NR=1{print $3}')

 HAPROXYSTATS=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v)

--- a/hooks/charmhelpers/contrib/openstack/ha/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/ha/utils.py
@ -23,6 +23,8 @@
 Helpers for high availability.
 """

+import json
+
 import re

 from charmhelpers.core.hookenv import (
@ -32,6 +34,7 @@ from charmhelpers.core.hookenv import (
    config,
    status_set,
    DEBUG,
+    WARNING,
 )

 from charmhelpers.core.host import (
@ -40,6 +43,23 @@ from charmhelpers.core.host import (

 from charmhelpers.contrib.openstack.ip import (
    resolve_address,
+    is_ipv6,
+)
+
+from charmhelpers.contrib.network.ip import (
+    get_iface_for_address,
+    get_netmask_for_address,
+)
+
+from charmhelpers.contrib.hahelpers.cluster import (
+    get_hacluster_config
+)
+
+JSON_ENCODE_OPTIONS = dict(
+    sort_keys=True,
+    allow_nan=False,
+    indent=None,
+    separators=(',', ':'),
 )


@ -53,8 +73,8 @@ class DNSHAException(Exception):
 def update_dns_ha_resource_params(resources, resource_params,
                                  relation_id=None,
                                  crm_ocf='ocf:maas:dns'):
-    """ Check for os-*-hostname settings and update resource dictionaries for
-    the HA relation.
+    """ Configure DNS-HA resources based on provided configuration and
+    update resource dictionaries for the HA relation.

    @param resources: Pointer to dictionary of resources.
                      Usually instantiated in ha_joined().
@ -64,7 +84,85 @@ def update_dns_ha_resource_params(resources, resource_params,
    @param crm_ocf: Corosync Open Cluster Framework resource agent to use for
                    DNS HA
    """
+    _relation_data = {'resources': {}, 'resource_params': {}}
+    update_hacluster_dns_ha(charm_name(),
+                            _relation_data,
+                            crm_ocf)
+    resources.update(_relation_data['resources'])
+    resource_params.update(_relation_data['resource_params'])
+    relation_set(relation_id=relation_id, groups=_relation_data['groups'])

+
+def assert_charm_supports_dns_ha():
+    """Validate prerequisites for DNS HA
+    The MAAS client is only available on Xenial or greater
+
+    :raises DNSHAException: if release is < 16.04
+    """
+    if lsb_release().get('DISTRIB_RELEASE') < '16.04':
+        msg = ('DNS HA is only supported on 16.04 and greater '
+               'versions of Ubuntu.')
+        status_set('blocked', msg)
+        raise DNSHAException(msg)
+    return True
+
+
+def expect_ha():
+    """ Determine if the unit expects to be in HA
+
+    Check for VIP or dns-ha settings which indicate the unit should expect to
+    be related to hacluster.
+
+    @returns boolean
+    """
+    return config('vip') or config('dns-ha')
+
+
+def generate_ha_relation_data(service):
+    """ Generate relation data for ha relation
+
+    Based on configuration options and unit interfaces, generate a json
+    encoded dict of relation data items for the hacluster relation,
+    providing configuration for DNS HA or VIP's + haproxy clone sets.
+
+    @returns dict: json encoded data for use with relation_set
+    """
+    _haproxy_res = 'res_{}_haproxy'.format(service)
+    _relation_data = {
+        'resources': {
+            _haproxy_res: 'lsb:haproxy',
+        },
+        'resource_params': {
+            _haproxy_res: 'op monitor interval="5s"'
+        },
+        'init_services': {
+            _haproxy_res: 'haproxy'
+        },
+        'clones': {
+            'cl_{}_haproxy'.format(service): _haproxy_res
+        },
+    }
+
+    if config('dns-ha'):
+        update_hacluster_dns_ha(service, _relation_data)
+    else:
+        update_hacluster_vip(service, _relation_data)
+
+    return {
+        'json_{}'.format(k): json.dumps(v, **JSON_ENCODE_OPTIONS)
+        for k, v in _relation_data.items() if v
+    }
+
+
+def update_hacluster_dns_ha(service, relation_data,
+                            crm_ocf='ocf:maas:dns'):
+    """ Configure DNS-HA resources based on provided configuration
+
+    @param service: Name of the service being configured
+    @param relation_data: Pointer to dictionary of relation data.
+    @param crm_ocf: Corosync Open Cluster Framework resource agent to use for
+                    DNS HA
+    """
    # Validate the charm environment for DNS HA
    assert_charm_supports_dns_ha()

@ -93,7 +191,7 @@ def update_dns_ha_resource_params(resources, resource_params,
            status_set('blocked', msg)
            raise DNSHAException(msg)

-        hostname_key = 'res_{}_{}_hostname'.format(charm_name(), endpoint_type)
+        hostname_key = 'res_{}_{}_hostname'.format(service, endpoint_type)
        if hostname_key in hostname_group:
            log('DNS HA: Resource {}: {} already exists in '
                'hostname group - skipping'.format(hostname_key, hostname),
@ -101,42 +199,67 @@ def update_dns_ha_resource_params(resources, resource_params,
            continue

        hostname_group.append(hostname_key)
-        resources[hostname_key] = crm_ocf
-        resource_params[hostname_key] = (
-            'params fqdn="{}" ip_address="{}" '
-            ''.format(hostname, resolve_address(endpoint_type=endpoint_type,
-                                                override=False)))
+        relation_data['resources'][hostname_key] = crm_ocf
+        relation_data['resource_params'][hostname_key] = (
+            'params fqdn="{}" ip_address="{}"'
+            .format(hostname, resolve_address(endpoint_type=endpoint_type,
+                                              override=False)))

    if len(hostname_group) >= 1:
        log('DNS HA: Hostname group is set with {} as members. '
            'Informing the ha relation'.format(' '.join(hostname_group)),
            DEBUG)
-        relation_set(relation_id=relation_id, groups={
-            'grp_{}_hostnames'.format(charm_name()): ' '.join(hostname_group)})
+        relation_data['groups'] = {
+            'grp_{}_hostnames'.format(service): ' '.join(hostname_group)
+        }
    else:
        msg = 'DNS HA: Hostname group has no members.'
        status_set('blocked', msg)
        raise DNSHAException(msg)


-def assert_charm_supports_dns_ha():
-    """Validate prerequisites for DNS HA
-    The MAAS client is only available on Xenial or greater
+def update_hacluster_vip(service, relation_data):
+    """ Configure VIP resources based on provided configuration
+
+    @param service: Name of the service being configured
+    @param relation_data: Pointer to dictionary of relation data.
    """
-    if lsb_release().get('DISTRIB_RELEASE') < '16.04':
-        msg = ('DNS HA is only supported on 16.04 and greater '
-               'versions of Ubuntu.')
-        status_set('blocked', msg)
-        raise DNSHAException(msg)
-    return True
+    cluster_config = get_hacluster_config()
+    vip_group = []
+    for vip in cluster_config['vip'].split():
+        if is_ipv6(vip):
+            res_neutron_vip = 'ocf:heartbeat:IPv6addr'
+            vip_params = 'ipv6addr'
+        else:
+            res_neutron_vip = 'ocf:heartbeat:IPaddr2'
+            vip_params = 'ip'

+        iface = (get_iface_for_address(vip) or
+                 config('vip_iface'))
+        netmask = (get_netmask_for_address(vip) or
+                   config('vip_cidr'))

-def expect_ha():
-    """ Determine if the unit expects to be in HA
+        if iface is not None:
+            vip_key = 'res_{}_{}_vip'.format(service, iface)
+            if vip_key in vip_group:
+                if vip not in relation_data['resource_params'][vip_key]:
+                    vip_key = '{}_{}'.format(vip_key, vip_params)
+                else:
+                    log("Resource '%s' (vip='%s') already exists in "
+                        "vip group - skipping" % (vip_key, vip), WARNING)
+                    continue

-    Check for VIP or dns-ha settings which indicate the unit should expect to
-    be related to hacluster.
+            relation_data['resources'][vip_key] = res_neutron_vip
+            relation_data['resource_params'][vip_key] = (
+                'params {ip}="{vip}" cidr_netmask="{netmask}" '
+                'nic="{iface}"'.format(ip=vip_params,
+                                       vip=vip,
+                                       iface=iface,
+                                       netmask=netmask)
+            )
+            vip_group.append(vip_key)

-    @returns boolean
-    """
-    return config('vip') or config('dns-ha')
+    if len(vip_group) >= 1:
+        relation_data['groups'] = {
+            'grp_{}_vips'.format(service): ' '.join(vip_group)
+        }
--- a/hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf
+++ b/hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf
@ -15,9 +15,6 @@ Listen {{ public_port }}
 {% if port -%}
 <VirtualHost *:{{ port }}>
    WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
-{% if python_path -%}
-                      python-path={{ python_path }} \
-{% endif -%}
                      display-name=%{GROUP}
    WSGIProcessGroup {{ service_name }}
    WSGIScriptAlias / {{ script }}
@ -29,7 +26,7 @@ Listen {{ public_port }}
    ErrorLog /var/log/apache2/{{ service_name }}_error.log
    CustomLog /var/log/apache2/{{ service_name }}_access.log combined

-    <Directory {{ usr_bin }}>
+    <Directory /usr/bin>
        <IfVersion >= 2.4>
            Require all granted
        </IfVersion>
@ -44,9 +41,6 @@ Listen {{ public_port }}
 {% if admin_port -%}
 <VirtualHost *:{{ admin_port }}>
    WSGIDaemonProcess {{ service_name }}-admin processes={{ admin_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
-{% if python_path -%}
-                      python-path={{ python_path }} \
-{% endif -%}
                      display-name=%{GROUP}
    WSGIProcessGroup {{ service_name }}-admin
    WSGIScriptAlias / {{ admin_script }}
@ -58,7 +52,7 @@ Listen {{ public_port }}
    ErrorLog /var/log/apache2/{{ service_name }}_error.log
    CustomLog /var/log/apache2/{{ service_name }}_access.log combined

-    <Directory {{ usr_bin }}>
+    <Directory /usr/bin>
        <IfVersion >= 2.4>
            Require all granted
        </IfVersion>
@ -73,9 +67,6 @@ Listen {{ public_port }}
 {% if public_port -%}
 <VirtualHost *:{{ public_port }}>
    WSGIDaemonProcess {{ service_name }}-public processes={{ public_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
-{% if python_path -%}
-                      python-path={{ python_path }} \
-{% endif -%}
                      display-name=%{GROUP}
    WSGIProcessGroup {{ service_name }}-public
    WSGIScriptAlias / {{ public_script }}
@ -87,7 +78,7 @@ Listen {{ public_port }}
    ErrorLog /var/log/apache2/{{ service_name }}_error.log
    CustomLog /var/log/apache2/{{ service_name }}_access.log combined

-    <Directory {{ usr_bin }}>
+    <Directory /usr/bin>
        <IfVersion >= 2.4>
            Require all granted
        </IfVersion>
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@ -626,11 +626,6 @@ def openstack_upgrade_available(package):
    else:
        avail_vers = get_os_version_install_source(src)
    apt.init()
-    if "swift" in package:
-        major_cur_vers = cur_vers.split('.', 1)[0]
-        major_avail_vers = avail_vers.split('.', 1)[0]
-        major_diff = apt.version_compare(major_avail_vers, major_cur_vers)
-        return avail_vers > cur_vers and (major_diff == 1 or major_diff == 0)
    return apt.version_compare(avail_vers, cur_vers) == 1


--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@ -39,6 +39,7 @@ if not six.PY3:
 else:
    from collections import UserDict

+
 CRITICAL = "CRITICAL"
 ERROR = "ERROR"
 WARNING = "WARNING"
@ -344,6 +345,7 @@ class Config(dict):

        """
        with open(self.path, 'w') as f:
+            os.fchmod(f.fileno(), 0o600)
            json.dump(self, f)

    def _implicit_save(self):
--- a/hooks/charmhelpers/core/unitdata.py
+++ b/hooks/charmhelpers/core/unitdata.py
@ -175,6 +175,8 @@ class Storage(object):
            else:
                self.db_path = os.path.join(
                    os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+        with open(self.db_path, 'a') as f:
+            os.fchmod(f.fileno(), 0o600)
        self.conn = sqlite3.connect('%s' % self.db_path)
        self.cursor = self.conn.cursor()
        self.revision = None
--- a/tests/charmhelpers/core/hookenv.py
+++ b/tests/charmhelpers/core/hookenv.py
@ -39,6 +39,7 @@ if not six.PY3:
 else:
    from collections import UserDict

+
 CRITICAL = "CRITICAL"
 ERROR = "ERROR"
 WARNING = "WARNING"
@ -344,6 +345,7 @@ class Config(dict):

        """
        with open(self.path, 'w') as f:
+            os.fchmod(f.fileno(), 0o600)
            json.dump(self, f)

    def _implicit_save(self):
--- a/tests/charmhelpers/core/unitdata.py
+++ b/tests/charmhelpers/core/unitdata.py
@ -175,6 +175,8 @@ class Storage(object):
            else:
                self.db_path = os.path.join(
                    os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+        with open(self.db_path, 'a') as f:
+            os.fchmod(f.fileno(), 0o600)
        self.conn = sqlite3.connect('%s' % self.db_path)
        self.cursor = self.conn.cursor()
        self.revision = None