[james-page,r=gnuoy,r=*] Refactor hacluster charm

1) supports reconfiguration of cluster resources from principle charm

2) direct configuration of mcastport and bindiface via juju configuration

3) quorum policy based on expected size of cluster

    2 = ignore quorum loss
    3 = stop on quorum loss

4) conditional restarting of corosync/pacemaker as required.

It's all just a bit nicer to use now!

README.md

@@ -34,13 +34,17 @@ in order for clustering to occur - otherwise nothing actually get configured.
 The hacluster interface supports a number of different cluster configuration
 options.
 
-## Mandatory Relation Data
+## Mandatory Relation Data (deprecated)
 
-All principle charms must provide basic corosync configuration:
+Principle charms should provide basic corosync configuration:
 
     corosync\_bindiface: The network interface to use for cluster messaging.
     corosync\_mcastport: The multicast port to use for cluster messaging.
 
+however, these can also be provided via configuration on the hacluster charm
+itself.  If configuration is provided directly to the hacluster charm, this
+will be preferred over these relation options from the principle charm.
+
 ## Resource Configuration
 
 The hacluster interface provides support for a number of different ways

config.yaml

@@ -6,6 +6,18 @@ options:
       Multicast IP address to use for exchanging messages over the network.
       If multiple clusters are on the same bindnetaddr network, this value
       can be changed.
+  corosync_bindiface:
+    type: string
+    default:
+    description: |
+      Default network interface on which HA cluster will bind to communication
+      with the other members of the HA Cluster.
+  corosync_mcastport:
+    type: int
+    default:
+    description: |
+      Default multicast port number that will be used to communicate between
+      HA Cluster nodes.
   corosync_key:
     type: string
     default: "64RxJNcCkwo8EJYBsaacitUvbQp5AW4YolJi5/2urYZYp2jfLxY+3IUCOaAUJHPle4Yqfy+WBXO0I/6ASSAjj9jaiHVNaxmVhhjcmyBqy2vtPf+m+0VxVjUXlkTyYsODwobeDdO3SIkbIABGfjLTu29yqPTsfbvSYr6skRb9ne0="
@@ -27,16 +39,25 @@ options:
      parameters are properly configured in its invenvory.
   maas_url:
     type: string
+    default:
     description: MAAS API endpoint (required for STONITH).
   maas_credentials:
     type: string
+    default:
     description: MAAS credentials (required for STONITH).
   cluster_count:
     type: int
     default: 2
-    description: Number of peer units required to bootstrap cluster services.
+    description: |
+      Number of peer units required to bootstrap cluster services.
+      .
+      If less that 3 is specified, the cluster will be configured to
+      ignore any quorum problems; with 3 or more units, quorum will be
+      enforced and services will be stopped in the event of a loss
+      of quorum.
   monitor_host:
     type: string
+    default:
     description: |
      One or more IPs, separated by space, that will be used as a saftey check
      for avoiding split brain situations. Nodes in the cluster will ping these

hooks/charmhelpers/contrib/network/ip.py

@@ -57,6 +57,8 @@ def get_address_in_network(network, fallback=None, fatal=False):
         else:
             if fatal:
                 not_found_error_out()
+            else:
+                return None
 
     _validate_cidr(network)
     network = netaddr.IPNetwork(network)

hooks/hooks.py

@@ -7,6 +7,7 @@
 #  Andres Rodriguez <andres.rodriguez@canonical.com>
 #
 
+import ast
 import shutil
 import sys
 import time
@@ -16,6 +17,7 @@ from base64 import b64decode
 import maas as MAAS
 import pcmk
 import hacluster
+import socket
 
 from charmhelpers.core.hookenv import (
     log,
@@ -34,11 +36,15 @@ from charmhelpers.core.host import (
     service_start,
     service_restart,
     service_running,
+    write_file,
+    mkdir,
+    file_hash,
     lsb_release
 )
 
 from charmhelpers.fetch import (
     apt_install,
+    apt_purge
 )
 
 from charmhelpers.contrib.hahelpers.cluster import (
@@ -48,21 +54,31 @@ from charmhelpers.contrib.hahelpers.cluster import (
 
 hooks = Hooks()
 
+COROSYNC_CONF = '/etc/corosync/corosync.conf'
+COROSYNC_DEFAULT = '/etc/default/corosync'
+COROSYNC_AUTHKEY = '/etc/corosync/authkey'
+
+COROSYNC_CONF_FILES = [
+    COROSYNC_DEFAULT,
+    COROSYNC_AUTHKEY,
+    COROSYNC_CONF
+]
+
+PACKAGES = ['corosync', 'pacemaker', 'python-netaddr', 'ipmitool']
+
 
 @hooks.hook()
 def install():
-    apt_install(['corosync', 'pacemaker', 'python-netaddr', 'ipmitool'],
+    apt_install(PACKAGES, fatal=True)
-                fatal=True)
+    # NOTE(adam_g) rbd OCF only included with newer versions of
-    # XXX rbd OCF only included with newer versions of ceph-resource-agents.
+    # ceph-resource-agents. Bundle /w charm until we figure out a
-    # Bundle /w charm until we figure out a better way to install it.
+    # better way to install it.
-    if not os.path.exists('/usr/lib/ocf/resource.d/ceph'):
+    mkdir('/usr/lib/ocf/resource.d/ceph')
-        os.makedirs('/usr/lib/ocf/resource.d/ceph')
     if not os.path.isfile('/usr/lib/ocf/resource.d/ceph/rbd'):
         shutil.copy('ocf/ceph/rbd', '/usr/lib/ocf/resource.d/ceph/rbd')
 
 
 def get_corosync_conf():
-    conf = {}
     if config('prefer-ipv6'):
         ip_version = 'ipv6'
         bindnetaddr = hacluster.get_ipv6_network_address
@@ -70,6 +86,19 @@ def get_corosync_conf():
         ip_version = 'ipv4'
         bindnetaddr = hacluster.get_network_address
 
+    # NOTE(jamespage) use local charm configuration over any provided by
+    # principle charm
+    conf = {
+        'corosync_bindnetaddr':
+        bindnetaddr(config('corosync_bindiface')),
+        'corosync_mcastport': config('corosync_mcastport'),
+        'corosync_mcastaddr': config('corosync_mcastaddr'),
+        'ip_version': ip_version,
+    }
+    if None not in conf.itervalues():
+        return conf
+
+    conf = {}
     for relid in relation_ids('ha'):
         for unit in related_units(relid):
             bindiface = relation_get('corosync_bindiface',
@@ -91,31 +120,35 @@ def get_corosync_conf():
             if None not in conf.itervalues():
                 return conf
     missing = [k for k, v in conf.iteritems() if v is None]
-    log('Missing required principle configuration: %s' % missing)
+    log('Missing required configuration: %s' % missing)
     return None
 
 
 def emit_corosync_conf():
-    # read config variables
     corosync_conf_context = get_corosync_conf()
-    # write config file (/etc/corosync/corosync.conf
+    if corosync_conf_context:
-    with open('/etc/corosync/corosync.conf', 'w') as corosync_conf:
+        write_file(path=COROSYNC_CONF,
-        corosync_conf.write(render_template('corosync.conf',
+                   content=render_template('corosync.conf',
-                                            corosync_conf_context))
+                                           corosync_conf_context))
+        return True
+    else:
+        return False
 
 
 def emit_base_conf():
     corosync_default_context = {'corosync_enabled': 'yes'}
-    # write /etc/default/corosync file
+    write_file(path=COROSYNC_DEFAULT,
-    with open('/etc/default/corosync', 'w') as corosync_default:
+               content=render_template('corosync',
-        corosync_default.write(render_template('corosync',
+                                       corosync_default_context))
-                                               corosync_default_context))
+
     corosync_key = config('corosync_key')
     if corosync_key:
-        # write the authkey
+        write_file(path=COROSYNC_AUTHKEY,
-        with open('/etc/corosync/authkey', 'w') as corosync_key_file:
+                   content=b64decode(corosync_key),
-            corosync_key_file.write(b64decode(corosync_key))
+                   perms=0o400)
-        os.chmod = ('/etc/corosync/authkey', 0o400)
+        return True
+    else:
+        return False
 
 
 @hooks.hook()
@@ -131,20 +164,16 @@ def config_changed():
 
     hacluster.enable_lsb_services('pacemaker')
 
-    # Create a new config file
+    if configure_corosync():
-    emit_base_conf()
+        pcmk.wait_for_pcmk()
-
+        configure_cluster_global()
-    # Reconfigure the cluster if required
+        configure_monitor_host()
-    configure_cluster()
+        configure_stonith()
-
-    # Setup fencing.
-    configure_stonith()
 
 
 @hooks.hook()
 def upgrade_charm():
     install()
-    config_changed()
 
 
 def restart_corosync():
@@ -154,82 +183,138 @@ def restart_corosync():
     time.sleep(5)
     service_start("pacemaker")
 
-HAMARKER = '/var/lib/juju/haconfigured'
+
+def restart_corosync_on_change():
+    '''Simple decorator to restart corosync if any of its config changes'''
+    def wrap(f):
+        def wrapped_f(*args):
+            checksums = {}
+            for path in COROSYNC_CONF_FILES:
+                checksums[path] = file_hash(path)
+            return_data = f(*args)
+            # NOTE: this assumes that this call is always done around
+            # configure_corosync, which returns true if configuration
+            # files where actually generated
+            if return_data:
+                for path in COROSYNC_CONF_FILES:
+                    if checksums[path] != file_hash(path):
+                        restart_corosync()
+                        break
+            return return_data
+        return wrapped_f
+    return wrap
+
+
+@restart_corosync_on_change()
+def configure_corosync():
+    log('Configuring and (maybe) restarting corosync')
+    return emit_base_conf() and emit_corosync_conf()
+
+
+def configure_monitor_host():
+    '''Configure extra monitor host for better network failure detection'''
+    log('Checking monitor host configuration')
+    monitor_host = config('monitor_host')
+    if monitor_host:
+        if not pcmk.crm_opt_exists('ping'):
+            log('Implementing monitor host'
+                ' configuration (host: %s)' % monitor_host)
+            monitor_interval = config('monitor_interval')
+            cmd = 'crm -w -F configure primitive ping' \
+                ' ocf:pacemaker:ping params host_list="%s"' \
+                ' multiplier="100" op monitor interval="%s"' %\
+                (monitor_host, monitor_interval)
+            pcmk.commit(cmd)
+            cmd = 'crm -w -F configure clone cl_ping ping' \
+                ' meta interleave="true"'
+            pcmk.commit(cmd)
+        else:
+            log('Reconfiguring monitor host'
+                ' configuration (host: %s)' % monitor_host)
+            cmd = 'crm -w -F resource param ping set host_list="%s"' %\
+                monitor_host
+    else:
+        if pcmk.crm_opt_exists('ping'):
+            log('Disabling monitor host configuration')
+            pcmk.commit('crm -w -F resource stop ping')
+            pcmk.commit('crm -w -F configure delete ping')
+
+
+def configure_cluster_global():
+    '''Configure global cluster options'''
+    log('Applying global cluster configuration')
+    if int(config('cluster_count')) >= 3:
+        # NOTE(jamespage) if 3 or more nodes, then quorum can be
+        # managed effectively, so stop if quorum lost
+        log('Configuring no-quorum-policy to stop')
+        cmd = "crm configure property no-quorum-policy=stop"
+    else:
+        # NOTE(jamespage) if less that 3 nodes, quorum not possible
+        # so ignore
+        log('Configuring no-quorum-policy to ignore')
+        cmd = "crm configure property no-quorum-policy=ignore"
+    pcmk.commit(cmd)
+
+    cmd = 'crm configure rsc_defaults $id="rsc-options"' \
+          ' resource-stickiness="100"'
+    pcmk.commit(cmd)
+
+
+def parse_data(relid, unit, key):
+    '''Simple helper to ast parse relation data'''
+    data = relation_get(key, unit, relid)
+    if data:
+        return ast.literal_eval(data)
+    else:
+        return {}
 
 
 @hooks.hook('ha-relation-joined',
             'ha-relation-changed',
             'hanode-relation-joined',
             'hanode-relation-changed')
-def configure_cluster():
+def configure_principle_cluster_resources():
-    # Check that we are not already configured
-    if os.path.exists(HAMARKER):
-        log('HA already configured, not reconfiguring')
-        return
     # Check that we are related to a principle and that
     # it has already provided the required corosync configuration
     if not get_corosync_conf():
-        log('Unable to configure corosync right now, bailing')
+        log('Unable to configure corosync right now, deferring configuration')
         return
     else:
-        log('Ready to form cluster - informing peers')
+        if relation_ids('hanode'):
-        relation_set(relation_id=relation_ids('hanode')[0],
+            log('Ready to form cluster - informing peers')
-                     ready=True)
+            relation_set(relation_id=relation_ids('hanode')[0],
+                         ready=True)
+        else:
+            log('Ready to form cluster, but not related to peers just yet')
+            return
+
     # Check that there's enough nodes in order to perform the
     # configuration of the HA cluster
     if (len(get_cluster_nodes()) <
             int(config('cluster_count'))):
-        log('Not enough nodes in cluster, bailing')
+        log('Not enough nodes in cluster, deferring configuration')
         return
 
     relids = relation_ids('ha')
     if len(relids) == 1:  # Should only ever be one of these
         # Obtain relation information
         relid = relids[0]
-        unit = related_units(relid)[0]
+        units = related_units(relid)
-        log('Using rid {} unit {}'.format(relid, unit))
+        if len(units) < 1:
-        import ast
+            log('No principle unit found, deferring configuration')
-        resources = \
+            return
-            {} if relation_get("resources",
+        unit = units[0]
-                               unit, relid) is None \
+        log('Parsing cluster configuration'
-            else ast.literal_eval(relation_get("resources",
+            ' using rid: {}, unit: {}'.format(relid, unit))
-                                               unit, relid))
+        resources = parse_data(relid, unit, 'resources')
-        resource_params = \
+        delete_resources = parse_data(relid, unit, 'delete_resources')
-            {} if relation_get("resource_params",
+        resource_params = parse_data(relid, unit, 'resource_params')
-                               unit, relid) is None \
+        groups = parse_data(relid, unit, 'groups')
-            else ast.literal_eval(relation_get("resource_params",
+        ms = parse_data(relid, unit, 'ms')
-                                               unit, relid))
+        orders = parse_data(relid, unit, 'orders')
-        groups = \
+        colocations = parse_data(relid, unit, 'colocations')
-            {} if relation_get("groups",
+        clones = parse_data(relid, unit, 'clones')
-                               unit, relid) is None \
+        init_services = parse_data(relid, unit, 'init_services')
-            else ast.literal_eval(relation_get("groups",
-                                               unit, relid))
-        ms = \
-            {} if relation_get("ms",
-                               unit, relid) is None \
-            else ast.literal_eval(relation_get("ms",
-                                               unit, relid))
-        orders = \
-            {} if relation_get("orders",
-                               unit, relid) is None \
-            else ast.literal_eval(relation_get("orders",
-                                               unit, relid))
-        colocations = \
-            {} if relation_get("colocations",
-                               unit, relid) is None \
-            else ast.literal_eval(relation_get("colocations",
-                                               unit, relid))
-        clones = \
-            {} if relation_get("clones",
-                               unit, relid) is None \
-            else ast.literal_eval(relation_get("clones",
-                                               unit, relid))
-        init_services = \
-            {} if relation_get("init_services",
-                               unit, relid) is None \
-            else ast.literal_eval(relation_get("init_services",
-                                               unit, relid))
-
     else:
         log('Related to {} ha services'.format(len(relids)))
         return
@@ -241,39 +326,26 @@ def configure_cluster():
                 for ra in resources.itervalues()]:
         apt_install('ceph-resource-agents')
 
-    log('Configuring and restarting corosync')
+    # NOTE: this should be removed in 15.04 cycle as corosync
-    emit_corosync_conf()
+    # configuration should be set directly on subordinate
-    restart_corosync()
+    configure_corosync()
-
-    log('Waiting for PCMK to start')
     pcmk.wait_for_pcmk()
-
+    configure_cluster_global()
-    log('Doing global cluster configuration')
+    configure_monitor_host()
-    cmd = "crm configure property stonith-enabled=false"
+    configure_stonith()
-    pcmk.commit(cmd)
-    cmd = "crm configure property no-quorum-policy=ignore"
-    pcmk.commit(cmd)
-    cmd = 'crm configure rsc_defaults $id="rsc-options"' \
-          ' resource-stickiness="100"'
-    pcmk.commit(cmd)
-
-    # Configure Ping service
-    monitor_host = config('monitor_host')
-    if monitor_host:
-        if not pcmk.crm_opt_exists('ping'):
-            monitor_interval = config('monitor_interval')
-            cmd = 'crm -w -F configure primitive ping' \
-                  ' ocf:pacemaker:ping params host_list="%s"' \
-                  ' multiplier="100" op monitor interval="%s"' %\
-                  (monitor_host, monitor_interval)
-            cmd2 = 'crm -w -F configure clone cl_ping ping' \
-                   ' meta interleave="true"'
-            pcmk.commit(cmd)
-            pcmk.commit(cmd2)
 
     # Only configure the cluster resources
     # from the oldest peer unit.
     if oldest_peer(peer_units()):
+        log('Deleting Resources')
+        log(str(delete_resources))
+        for res_name in delete_resources:
+            if pcmk.crm_opt_exists(res_name):
+                log('Stopping and deleting resource %s' % res_name)
+                if pcmk.crm_res_running(res_name):
+                    pcmk.commit('crm -w -F resource stop %s' % res_name)
+                pcmk.commit('crm -w -F configure delete %s' % res_name)
+
         log('Configuring Resources')
         log(str(resources))
         for res_name, res_type in resources.iteritems():
@@ -301,7 +373,7 @@ def configure_cluster():
                          resource_params[res_name])
                 pcmk.commit(cmd)
                 log('%s' % cmd)
-                if monitor_host:
+                if config('monitor_host'):
                     cmd = 'crm -F configure location Ping-%s %s rule' \
                           ' -inf: pingd lte 0' % (res_name, res_name)
                     pcmk.commit(cmd)
@@ -379,62 +451,55 @@ def configure_cluster():
         relation_set(relation_id=rel_id,
                      clustered="yes")
 
-    with open(HAMARKER, 'w') as marker:
-        marker.write('done')
-
-    configure_stonith()
-
 
 def configure_stonith():
-    if config('stonith_enabled') not in ['true', 'True']:
+    if config('stonith_enabled') not in ['true', 'True', True]:
-        return
+        log('Disabling STONITH')
-
+        cmd = "crm configure property stonith-enabled=false"
-    if not os.path.exists(HAMARKER):
+        pcmk.commit(cmd)
-        log('HA not yet configured, skipping STONITH config.')
+    else:
-        return
+        log('Enabling STONITH for all nodes in cluster.')
-
+        # configure stontih resources for all nodes in cluster.
-    log('Configuring STONITH for all nodes in cluster.')
+        # note: this is totally provider dependent and requires
-    # configure stontih resources for all nodes in cluster.
+        # access to the MAAS API endpoint, using endpoint and credentials
-    # note: this is totally provider dependent and requires
+        # set in config.
-    # access to the MAAS API endpoint, using endpoint and credentials
+        url = config('maas_url')
-    # set in config.
+        creds = config('maas_credentials')
-    url = config('maas_url')
+        if None in [url, creds]:
-    creds = config('maas_credentials')
+            log('maas_url and maas_credentials must be set'
-    if None in [url, creds]:
+                ' in config to enable STONITH.')
-        log('maas_url and maas_credentials must be set'
-            ' in config to enable STONITH.')
-        sys.exit(1)
-
-    maas = MAAS.MAASHelper(url, creds)
-    nodes = maas.list_nodes()
-    if not nodes:
-        log('Could not obtain node inventory from '
-            'MAAS @ %s.' % url)
-        sys.exit(1)
-
-    cluster_nodes = pcmk.list_nodes()
-    for node in cluster_nodes:
-        rsc, constraint = pcmk.maas_stonith_primitive(nodes, node)
-        if not rsc:
-            log('Failed to determine STONITH primitive for node'
-                ' %s' % node)
             sys.exit(1)
 
-        rsc_name = str(rsc).split(' ')[1]
+        maas = MAAS.MAASHelper(url, creds)
-        if not pcmk.is_resource_present(rsc_name):
+        nodes = maas.list_nodes()
-            log('Creating new STONITH primitive %s.' %
+        if not nodes:
-                rsc_name)
+            log('Could not obtain node inventory from '
-            cmd = 'crm -F configure %s' % rsc
+                'MAAS @ %s.' % url)
-            pcmk.commit(cmd)
+            sys.exit(1)
-            if constraint:
-                cmd = 'crm -F configure %s' % constraint
-                pcmk.commit(cmd)
-        else:
-            log('STONITH primitive already exists '
-                'for node.')
 
-    cmd = "crm configure property stonith-enabled=true"
+        cluster_nodes = pcmk.list_nodes()
-    pcmk.commit(cmd)
+        for node in cluster_nodes:
+            rsc, constraint = pcmk.maas_stonith_primitive(nodes, node)
+            if not rsc:
+                log('Failed to determine STONITH primitive for node'
+                    ' %s' % node)
+                sys.exit(1)
+
+            rsc_name = str(rsc).split(' ')[1]
+            if not pcmk.is_resource_present(rsc_name):
+                log('Creating new STONITH primitive %s.' %
+                    rsc_name)
+                cmd = 'crm -F configure %s' % rsc
+                pcmk.commit(cmd)
+                if constraint:
+                    cmd = 'crm -F configure %s' % constraint
+                    pcmk.commit(cmd)
+            else:
+                log('STONITH primitive already exists '
+                    'for node.')
+
+        cmd = "crm configure property stonith-enabled=true"
+        pcmk.commit(cmd)
 
 
 def get_cluster_nodes():
@@ -467,6 +532,13 @@ def render_template(template_name, context, template_dir=TEMPLATES_DIR):
     return template.render(context)
 
 
+@hooks.hook()
+def stop():
+    cmd = 'crm -w -F node delete %s' % socket.gethostname()
+    pcmk.commit(cmd)
+    apt_purge(['corosync', 'pacemaker'], fatal=True)
+
+
 def assert_charm_supports_ipv6():
     """Check whether we are able to support charms ipv6."""
     if lsb_release()['DISTRIB_CODENAME'].lower() < "trusty":