DNS HA

Allow DNS be the HA resource in leiu of a VIP when using MAAS 2.0.
Added an OCF resource dns
Added maas_dns.py as the api script to update a MAAS 2.0 DNS resource
record.

Charmhelpers sync to pull in DNS HA helpers

Change-Id: I0b71feec86a77643892fadc08f2954204b541d01

.gitignore

@@ -1,3 +1,5 @@
+bin/
 *.pyc
 .tox
 .testrepository
+.coverage

Makefile

@@ -3,7 +3,7 @@ PYTHON := /usr/bin/env python
 
 lint:
 	@flake8 --exclude hooks/charmhelpers,tests/charmhelpers \
-        hooks unit_tests tests
+        hooks unit_tests tests ocf/maas
 	@charm proof
 
 test:

README.md

@@ -29,6 +29,28 @@ To enable HA clustering support (for mysql for example):
 The principle charm must have explicit support for the hacluster interface
 in order for clustering to occur - otherwise nothing actually get configured.
 
+
+# HA/Clustering
+
+There are two mutually exclusive high availability options: using virtual
+IP(s) or DNS.
+
+To use virtual IP(s) the clustered nodes must be on the same subnet such that
+the VIP is a valid IP on the subnet for one of the node's interfaces and each
+node has an interface in said subnet. The VIP becomes a highly-available API
+endpoint.
+
+To use DNS high availability there are several prerequisites. However, DNS HA
+does not require the clustered nodes to be on the same subnet.
+Currently the DNS HA feature is only available for MAAS 2.0 or greater
+environments. MAAS 2.0 requires Juju 2.0 or greater. The MAAS 2.0 client
+requires Ubuntu 16.04 or greater. The clustered nodes must have static or
+"reserved" IP addresses registered in MAAS. The DNS hostname(s) must be
+pre-registered in MAAS before use with DNS HA.
+
+The charm will throw an exception in the following circumstances:
+If running on a version of Ubuntu less than Xenial 16.04
+
 # Usage for Charm Authors
 
 The hacluster interface supports a number of different cluster configuration

charm-helpers-hooks.yaml

@@ -8,7 +8,9 @@ include:
     - contrib.hahelpers
     - contrib.storage
     - contrib.network.ip
-    - contrib.openstack.utils
     - contrib.openstack.exceptions
+    - contrib.openstack.ip
+    - contrib.openstack.utils
+    - contrib.openstack.ha.utils
     - contrib.python.packages
     - contrib.charmsupport

config.yaml

@@ -61,6 +61,18 @@ options:
     type: string
     default:
     description: MAAS credentials (required for STONITH).
+  maas_source:
+     type: string
+     default: ppa:maas/stable
+     description: |
+       PPA for python3-maas-client:
+       .
+         - ppa:maas/stable
+         - ppa:maas/next
+       .
+       The last option should be used in conjunction with the key configuration
+       option.
+       Used when service_dns is set on the primary charm for DNS HA
   cluster_count:
     type: int
     default: 2

hooks/charmhelpers/contrib/hahelpers/cluster.py

@@ -280,14 +280,14 @@ def get_hacluster_config(exclude_keys=None):
     for initiating a relation to hacluster:
 
         ha-bindiface, ha-mcastport, vip, os-internal-hostname,
-        os-admin-hostname, os-public-hostname
+        os-admin-hostname, os-public-hostname, os-access-hostname
 
     param: exclude_keys: list of setting key(s) to be excluded.
     returns: dict: A dict containing settings keyed by setting name.
     raises: HAIncompleteConfig if settings are missing or incorrect.
     '''
     settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'os-internal-hostname',
-                'os-admin-hostname', 'os-public-hostname']
+                'os-admin-hostname', 'os-public-hostname', 'os-access-hostname']
     conf = {}
     for setting in settings:
         if exclude_keys and setting in exclude_keys:
@@ -324,7 +324,7 @@ def valid_hacluster_config():
     # If dns-ha then one of os-*-hostname must be set
     if dns:
         dns_settings = ['os-internal-hostname', 'os-admin-hostname',
-                        'os-public-hostname']
+                        'os-public-hostname', 'os-access-hostname']
         # At this point it is unknown if one or all of the possible
         # network spaces are in HA. Validate at least one is set which is
         # the minimum required.

hooks/charmhelpers/contrib/openstack/ha/utils.py

@@ -0,0 +1,130 @@
+# Copyright 2014-2016 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+#
+# Copyright 2016 Canonical Ltd.
+#
+# Authors:
+#  Openstack Charmers <
+#
+
+"""
+Helpers for high availability.
+"""
+
+import re
+
+from charmhelpers.core.hookenv import (
+    log,
+    relation_set,
+    charm_name,
+    config,
+    status_set,
+    DEBUG,
+)
+
+from charmhelpers.core.host import (
+    lsb_release
+)
+
+from charmhelpers.contrib.openstack.ip import (
+    resolve_address,
+)
+
+
+class DNSHAException(Exception):
+    """Raised when an error occurs setting up DNS HA
+    """
+
+    pass
+
+
+def update_dns_ha_resource_params(resources, resource_params,
+                                  relation_id=None,
+                                  crm_ocf='ocf:maas:dns'):
+    """ Check for os-*-hostname settings and update resource dictionaries for
+    the HA relation.
+
+    @param resources: Pointer to dictionary of resources.
+                      Usually instantiated in ha_joined().
+    @param resource_params: Pointer to dictionary of resource parameters.
+                            Usually instantiated in ha_joined()
+    @param relation_id: Relation ID of the ha relation
+    @param crm_ocf: Corosync Open Cluster Framework resource agent to use for
+                    DNS HA
+    """
+
+    # Validate the charm environment for DNS HA
+    assert_charm_supports_dns_ha()
+
+    settings = ['os-admin-hostname', 'os-internal-hostname',
+                'os-public-hostname', 'os-access-hostname']
+
+    # Check which DNS settings are set and update dictionaries
+    hostname_group = []
+    for setting in settings:
+        hostname = config(setting)
+        if hostname is None:
+            log('DNS HA: Hostname setting {} is None. Ignoring.'
+                ''.format(setting),
+                DEBUG)
+            continue
+        m = re.search('os-(.+?)-hostname', setting)
+        if m:
+            networkspace = m.group(1)
+        else:
+            msg = ('Unexpected DNS hostname setting: {}. '
+                   'Cannot determine network space name'
+                   ''.format(setting))
+            status_set('blocked', msg)
+            raise DNSHAException(msg)
+
+        hostname_key = 'res_{}_{}_hostname'.format(charm_name(), networkspace)
+        if hostname_key in hostname_group:
+            log('DNS HA: Resource {}: {} already exists in '
+                'hostname group - skipping'.format(hostname_key, hostname),
+                DEBUG)
+            continue
+
+        hostname_group.append(hostname_key)
+        resources[hostname_key] = crm_ocf
+        resource_params[hostname_key] = (
+            'params fqdn="{}" ip_address="{}" '
+            ''.format(hostname, resolve_address(endpoint_type=networkspace,
+                                                override=False)))
+
+    if len(hostname_group) >= 1:
+        log('DNS HA: Hostname group is set with {} as members. '
+            'Informing the ha relation'.format(' '.join(hostname_group)),
+            DEBUG)
+        relation_set(relation_id=relation_id, groups={
+            'grp_{}_hostnames'.format(charm_name()): ' '.join(hostname_group)})
+    else:
+        msg = 'DNS HA: Hostname group has no members.'
+        status_set('blocked', msg)
+        raise DNSHAException(msg)
+
+
+def assert_charm_supports_dns_ha():
+    """Validate prerequisites for DNS HA
+    The MAAS client is only available on Xenial or greater
+    """
+    if lsb_release().get('DISTRIB_RELEASE') < '16.04':
+        msg = ('DNS HA is only supported on 16.04 and greater '
+               'versions of Ubuntu.')
+        status_set('blocked', msg)
+        raise DNSHAException(msg)
+    return True

hooks/charmhelpers/contrib/openstack/ip.py

@@ -0,0 +1,182 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from charmhelpers.core.hookenv import (
+    config,
+    unit_get,
+    service_name,
+    network_get_primary_address,
+)
+from charmhelpers.contrib.network.ip import (
+    get_address_in_network,
+    is_address_in_network,
+    is_ipv6,
+    get_ipv6_addr,
+    resolve_network_cidr,
+)
+from charmhelpers.contrib.hahelpers.cluster import is_clustered
+
+PUBLIC = 'public'
+INTERNAL = 'int'
+ADMIN = 'admin'
+
+ADDRESS_MAP = {
+    PUBLIC: {
+        'binding': 'public',
+        'config': 'os-public-network',
+        'fallback': 'public-address',
+        'override': 'os-public-hostname',
+    },
+    INTERNAL: {
+        'binding': 'internal',
+        'config': 'os-internal-network',
+        'fallback': 'private-address',
+        'override': 'os-internal-hostname',
+    },
+    ADMIN: {
+        'binding': 'admin',
+        'config': 'os-admin-network',
+        'fallback': 'private-address',
+        'override': 'os-admin-hostname',
+    }
+}
+
+
+def canonical_url(configs, endpoint_type=PUBLIC):
+    """Returns the correct HTTP URL to this host given the state of HTTPS
+    configuration, hacluster and charm configuration.
+
+    :param configs: OSTemplateRenderer config templating object to inspect
+                    for a complete https context.
+    :param endpoint_type: str endpoint type to resolve.
+    :param returns: str base URL for services on the current service unit.
+    """
+    scheme = _get_scheme(configs)
+
+    address = resolve_address(endpoint_type)
+    if is_ipv6(address):
+        address = "[{}]".format(address)
+
+    return '%s://%s' % (scheme, address)
+
+
+def _get_scheme(configs):
+    """Returns the scheme to use for the url (either http or https)
+    depending upon whether https is in the configs value.
+
+    :param configs: OSTemplateRenderer config templating object to inspect
+                    for a complete https context.
+    :returns: either 'http' or 'https' depending on whether https is
+              configured within the configs context.
+    """
+    scheme = 'http'
+    if configs and 'https' in configs.complete_contexts():
+        scheme = 'https'
+    return scheme
+
+
+def _get_address_override(endpoint_type=PUBLIC):
+    """Returns any address overrides that the user has defined based on the
+    endpoint type.
+
+    Note: this function allows for the service name to be inserted into the
+    address if the user specifies {service_name}.somehost.org.
+
+    :param endpoint_type: the type of endpoint to retrieve the override
+                          value for.
+    :returns: any endpoint address or hostname that the user has overridden
+              or None if an override is not present.
+    """
+    override_key = ADDRESS_MAP[endpoint_type]['override']
+    addr_override = config(override_key)
+    if not addr_override:
+        return None
+    else:
+        return addr_override.format(service_name=service_name())
+
+
+def resolve_address(endpoint_type=PUBLIC, override=True):
+    """Return unit address depending on net config.
+
+    If unit is clustered with vip(s) and has net splits defined, return vip on
+    correct network. If clustered with no nets defined, return primary vip.
+
+    If not clustered, return unit address ensuring address is on configured net
+    split if one is configured, or a Juju 2.0 extra-binding has been used.
+
+    :param endpoint_type: Network endpoing type
+    :param override: Accept hostname overrides or not
+    """
+    resolved_address = None
+    if override:
+        resolved_address = _get_address_override(endpoint_type)
+        if resolved_address:
+            return resolved_address
+
+    vips = config('vip')
+    if vips:
+        vips = vips.split()
+
+    net_type = ADDRESS_MAP[endpoint_type]['config']
+    net_addr = config(net_type)
+    net_fallback = ADDRESS_MAP[endpoint_type]['fallback']
+    binding = ADDRESS_MAP[endpoint_type]['binding']
+    clustered = is_clustered()
+
+    if clustered and vips:
+        if net_addr:
+            for vip in vips:
+                if is_address_in_network(net_addr, vip):
+                    resolved_address = vip
+                    break
+        else:
+            # NOTE: endeavour to check vips against network space
+            #       bindings
+            try:
+                bound_cidr = resolve_network_cidr(
+                    network_get_primary_address(binding)
+                )
+                for vip in vips:
+                    if is_address_in_network(bound_cidr, vip):
+                        resolved_address = vip
+                        break
+            except NotImplementedError:
+                # If no net-splits configured and no support for extra
+                # bindings/network spaces so we expect a single vip
+                resolved_address = vips[0]
+    else:
+        if config('prefer-ipv6'):
+            fallback_addr = get_ipv6_addr(exc_list=vips)[0]
+        else:
+            fallback_addr = unit_get(net_fallback)
+
+        if net_addr:
+            resolved_address = get_address_in_network(net_addr, fallback_addr)
+        else:
+            # NOTE: only try to use extra bindings if legacy network
+            #       configuration is not in use
+            try:
+                resolved_address = network_get_primary_address(binding)
+            except NotImplementedError:
+                resolved_address = fallback_addr
+
+    if resolved_address is None:
+        raise ValueError("Unable to resolve a suitable IP address based on "
+                         "charm state and configuration. (net_type=%s, "
+                         "clustered=%s)" % (net_type, clustered))
+
+    return resolved_address

hooks/hooks.py

@@ -28,7 +28,6 @@ from charmhelpers.core.hookenv import (
 from charmhelpers.core.host import (
     service_stop,
     service_running,
-    mkdir,
 )
 
 from charmhelpers.fetch import (
@@ -55,6 +54,9 @@ from utils import (
     disable_lsb_services,
     disable_upstart_services,
     get_ipv6_addr,
+    validate_dns_ha,
+    setup_maas_api,
+    setup_ocf_files,
     set_unit_status,
 )
 
@@ -88,12 +90,7 @@ def install():
     # should be removed once the pacemaker package is fixed.
     status_set('maintenance', 'Installing apt packages')
     apt_install(filter_installed_packages(PACKAGES), fatal=True)
-    # NOTE(adam_g) rbd OCF only included with newer versions of
-    # ceph-resource-agents. Bundle /w charm until we figure out a
-    # better way to install it.
-    mkdir('/usr/lib/ocf/resource.d/ceph')
-    if not os.path.isfile('/usr/lib/ocf/resource.d/ceph/rbd'):
-        shutil.copy('ocf/ceph/rbd', '/usr/lib/ocf/resource.d/ceph/rbd')
+    setup_ocf_files()
 
 
 def get_transport():
@@ -121,6 +118,9 @@ def ensure_ipv6_requirements(hanode_rid):
 
 @hooks.hook()
 def config_changed():
+
+    setup_ocf_files()
+
     if config('prefer-ipv6'):
         assert_charm_supports_ipv6()
 
@@ -221,6 +221,25 @@ def ha_relation_changed():
                 for ra in resources.itervalues()]:
         apt_install('ceph-resource-agents')
 
+    if True in [ra.startswith('ocf:maas')
+                for ra in resources.values()]:
+        if validate_dns_ha():
+            log('Setting up access to MAAS API', level=INFO)
+            setup_maas_api()
+            # Update resource_parms for DNS resources to include MAAS URL and
+            # credentials
+            for resource in resource_params.keys():
+                if resource.endswith("_hostname"):
+                    resource_params[resource] += (
+                        ' maas_url="{}" maas_credentials="{}"'
+                        ''.format(config('maas_url'),
+                                  config('maas_credentials')))
+        else:
+            msg = ("DNS HA is requested but maas_url "
+                   "or maas_credentials are not set")
+            status_set('blocked', msg)
+            raise ValueError(msg)
+
     # NOTE: this should be removed in 15.04 cycle as corosync
     # configuration should be set directly on subordinate
     configure_corosync()

hooks/utils.py

@@ -30,7 +30,12 @@ from charmhelpers.contrib.openstack.utils import (
     clear_unit_paused,
     is_unit_paused_set,
 )
+from charmhelpers.contrib.openstack.ha.utils import (
+    assert_charm_supports_dns_ha
+)
 from charmhelpers.core.host import (
+    mkdir,
+    rsync,
     service_start,
     service_stop,
     service_restart,
@@ -41,6 +46,8 @@ from charmhelpers.core.host import (
 )
 from charmhelpers.fetch import (
     apt_install,
+    add_source,
+    apt_update,
 )
 from charmhelpers.contrib.hahelpers.cluster import (
     peer_ips,
@@ -82,6 +89,10 @@ COROSYNC_CONF_FILES = [
 SUPPORTED_TRANSPORTS = ['udp', 'udpu', 'multicast', 'unicast']
 
 
+class MAASConfigIncomplete(Exception):
+    pass
+
+
 def disable_upstart_services(*services):
     for service in services:
         with open("/etc/init/{}.override".format(service), "w") as override:
@@ -517,6 +528,50 @@ def restart_corosync():
         service_start("pacemaker")
 
 
+def validate_dns_ha():
+    """Validate the DNS HA
+
+    Assert the charm will support DNS HA
+    Check MAAS related configuration options are properly set
+    """
+
+    # Will raise an exception if unable to continue
+    assert_charm_supports_dns_ha()
+
+    if config('maas_url') and config('maas_credentials'):
+        return True
+    else:
+        msg = ("DNS HA is requested but the maas_url or maas_credentials "
+               "settings are not set")
+        status_set('blocked', msg)
+        raise MAASConfigIncomplete(msg)
+
+
+def setup_maas_api():
+    """Install MAAS PPA and packages for accessing the MAAS API.
+    """
+    add_source(config('maas_source'))
+    apt_update(fatal=True)
+    apt_install('python3-maas-client', fatal=True)
+
+
+def setup_ocf_files():
+    """Setup OCF resrouce agent files
+    """
+
+    # TODO (thedac) Eventually we want to package the OCF files.
+    # Bundle with the charm until then.
+    mkdir('/usr/lib/ocf/resource.d/ceph')
+    mkdir('/usr/lib/ocf/resource.d/maas')
+    # Xenial corosync is not creating this directory
+    mkdir('/etc/corosync/uidgid.d')
+
+    rsync('ocf/ceph/rbd', '/usr/lib/ocf/resource.d/ceph/rbd')
+    rsync('ocf/maas/dns', '/usr/lib/ocf/resource.d/maas/dns')
+    rsync('ocf/maas/maas_dns.py', '/usr/lib/heartbeat/maas_dns.py')
+    rsync('ocf/maas/maasclient/', '/usr/lib/heartbeat/maasclient/')
+
+
 def is_in_standby_mode(node_name):
     """Check if node is in standby mode in pacemaker
 

ocf/maas/dns

@@ -0,0 +1,267 @@
+#!/bin/sh
+#
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# OCF instance parameters
+#       OCF_RESKEY_logfile
+#       OCF_RESKEY_errlogfile
+#
+# This RA starts $binfile with $cmdline_options as $user in $workdir and writes a $pidfile from that.
+# If you want it to, it logs:
+# - stdout to $logfile, stderr to $errlogfile or
+# - stdout and stderr to $logfile
+# - or to will be captured by lrmd if these options are omitted.
+# Monitoring is done through $pidfile or your custom $monitor_hook script.
+# The RA expects the program to keep running "daemon-like" and
+# not just quit and exit. So this is NOT (yet - feel free to
+# enhance) a way to just run a single one-shot command which just
+# does something and then exits.
+
+
+# XXX Update all comments
+
+# Initialization:
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+#    OCF parameters are as below
+#    OCF_RESKEY_fqdn
+#    OCF_RESKEY_ip_address
+#    OCF_RESKEY_ttl
+#    OCF_RESKEY_maas_url
+#    OCF_RESKEY_maas_credentials
+
+# Defaults
+
+
+maas_dns_usage() {
+cat <<END
+usage: $0 {start|stop|monitor|validate-all|meta-data}
+Expects to have a fully populated OCF RA-compliant environment set.
+END
+}
+
+
+# Do we already serve this IP address on the given $NIC?
+#
+# returns:
+# ok = served (for CIP: + hash bucket)
+# partial = served and no hash bucket (CIP only)
+# partial2 = served and no CIP iptables rule
+# no = nothing
+#
+dns_served() {
+	target=`dig +short $OCF_RESKEY_fqdn`
+	if [ "x$target" != "x" ]
+	then
+		if test "$OCF_RESKEY_ip_address" = "$target"
+		then
+			echo "ok"
+			return 0
+		else
+			echo "no"
+			return 0
+		fi
+
+	else
+		echo "no"
+		return 0
+	fi
+}
+
+maas_dns_status() {
+	local dns_status=`dns_served`
+	if [ "$dns_status" = "ok" ]; then
+		return 0
+	fi
+
+}
+
+maas_dns_start() {
+	echo "maas_dns_start"
+	local dns_status=`dns_served`
+	if [ "$dns_status" = "ok" ]; then
+		exit $OCF_SUCCESS
+	fi
+
+	cmd="python3 $binfile --fqdn=$OCF_RESKEY_fqdn --ip_address=$OCF_RESKEY_ip_address --maas_server=$OCF_RESKEY_maas_url --maas_credentials=$OCF_RESKEY_maas_credentials "
+	if [ -n "$OCF_RESKEY_ttl" ]; then
+		cmd="$cmd --ttl=$OCF_RESKEY_ttl"
+	fi
+	ocf_log debug "Executing: $cmd"
+	# Execute the command as created above
+	eval $cmd
+	if [ $? -ne 0 ]; then
+		ocf_log err "$CMD failed."
+		exit $OCF_ERR_GENERIC
+	fi
+
+	exit $OCF_SUCCESS
+}
+
+maas_dns_stop() {
+	echo "maas_dns_stop"
+	local dns_status=`dns_served`
+	if [ $dns_status = "no" ]; then
+		: Requested interface not in use
+		exit $OCF_SUCCESS
+	fi
+
+	#XXX Should we remove the Entry?
+	# Code to "stop" the dns entry
+	sed -i "/$OCF_RESKEY_fqdn/d" /etc/hosts
+	exit $OCF_SUCCESS
+}
+
+maas_dns_monitor() {
+	echo "maas_dns_monitor"
+	local dns_status=`dns_served`
+	case $dns_status in
+	ok)
+		return $OCF_SUCCESS
+		;;
+	no)
+		exit 7
+		exit $OCF_NOT_RUNNING
+		;;
+	*)
+		# Errors
+		return $OCF_ERR_GENERIC
+		;;
+	esac
+
+}
+
+binfile="$HA_BIN/maas_dns.py"
+logfile="$OCF_RESKEY_logfile"
+errlogfile="$OCF_RESKEY_errlogfile"
+user="$OCF_RESKEY_user"
+[ -z "$user" ] && user=root
+
+maas_dns_validate() {
+	echo "maas_dns_validate"
+	if ! su - $user -c "test -x $binfile"
+	then
+		ocf_log err "$binfile does not exist or is not executable."
+		exit $OCF_ERR_INSTALLED
+	fi
+	if ! getent passwd $user >/dev/null 2>&1
+	then
+		ocf_log err "user $user does not exist."
+		exit $OCF_ERR_INSTALLED
+	fi
+	for logfilename in "$logfile" "$errlogfile"
+	do
+		if [ -n "$logfilename" ]; then
+			mkdir -p `dirname $logfilename` || {
+				ocf_log err "cannot create $(dirname $logfilename)"
+				exit $OCF_ERR_INSTALLED
+			}
+		fi
+	done
+	return $OCF_SUCCESS
+}
+
+maas_dns_meta() {
+cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="maas_dns">
+<version>1.0</version>
+<longdesc lang="en">
+OCF RA to manage MAAS DNS entries. This will call out to maas_dns.py with the command name, fqdn, ip_address, maas_url and maas_credentials
+</longdesc>
+<shortdesc lang="en">OCF RA to manage MAAS DNS entries</shortdesc>
+<parameters>
+<parameter name="fqdn" required="1">
+<longdesc lang="en">
+The fully qualified domain name for the DNS entry.
+</longdesc>
+<shortdesc lang="en">Fully qualified domain name</shortdesc>
+<content type="string" default=""/>
+</parameter>
+<parameter name="ip_address" required="1">
+<longdesc lang="en">
+The IP address for the DNS entry
+</longdesc>
+<shortdesc lang="en">IP Address</shortdesc>
+<content type="string" />
+</parameter>
+<parameter name="maas_url" required="1">
+<longdesc lang="en">
+The URL to the MAAS host where the DNS entry will be updated.
+</longdesc>
+<shortdesc lang="en">MAAS URL</shortdesc>
+<content type="string" default=""/>
+</parameter>
+<parameter name="maas_credentials" required="1">
+<longdesc lang="en">
+MAAS Oauth credentials for the MAAS API
+</longdesc>
+<shortdesc lang="en">MAAS Credentials</shortdesc>
+<content type="string" default=""/>
+</parameter>
+<parameter name="logfile" required="0">
+<longdesc lang="en">
+File to write STDOUT to
+</longdesc>
+<shortdesc lang="en">File to write STDOUT to</shortdesc>
+<content type="string" />
+</parameter>
+<parameter name="errlogfile" required="0">
+<longdesc lang="en">
+File to write STDERR to
+</longdesc>
+<shortdesc lang="en">File to write STDERR to</shortdesc>
+<content type="string" />
+</parameter>
+</parameters>
+<actions>
+<action name="start"   timeout="20s" />
+<action name="stop"    timeout="20s" />
+<action name="monitor" depth="0"  timeout="20s" interval="10" />
+<action name="meta-data"  timeout="5" />
+<action name="validate-all"  timeout="5" />
+</actions>
+</resource-agent>
+END
+exit 0
+}
+
+case "$1" in
+	meta-data|metadata|meta_data)
+		maas_dns_meta
+	;;
+	start)
+		maas_dns_start
+	;;
+	stop)
+		maas_dns_stop
+	;;
+	monitor)
+		maas_dns_monitor
+	;;
+	status)
+		maas_dns_status
+	;;
+	validate-all)
+		maas_dns_validate
+	;;
+	*)
+		maas_dns_usage
+		ocf_log err "$0 was called with unsupported arguments: $*"
+		exit $OCF_ERR_UNIMPLEMENTED
+	;;
+esac

ocf/maas/maas_dns.py

@@ -0,0 +1,166 @@
+#!/usr/bin/python3
+#
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import maasclient
+import argparse
+import sys
+import logging
+
+
+class MAASDNS(object):
+    def __init__(self, options):
+        self.maas = maasclient.MAASClient(options.maas_server,
+                                          options.maas_credentials)
+        # String representation of the fqdn
+        self.fqdn = options.fqdn
+        # Dictionary representation of MAAS dnsresource object
+        # TODO: Do this as a property
+        self.dnsresource = self.get_dnsresource()
+        # String representation of the time to live
+        self.ttl = str(options.ttl)
+        # String representation of the ip
+        self.ip = options.ip_address
+
+    def get_dnsresource(self):
+        """ Get a dnsresource object """
+        dnsresources = self.maas.get_dnsresources()
+        self.dnsresource = None
+        for dnsresource in dnsresources:
+            if dnsresource['fqdn'] == self.fqdn:
+                self.dnsresource = dnsresource
+        return self.dnsresource
+
+    def get_dnsresource_id(self):
+        """ Get a dnsresource ID """
+        return self.dnsresource['id']
+
+    def update_resource(self):
+        """ Update a dnsresource record with an IP """
+        return self.maas.update_dnsresource(self.dnsresource['id'],
+                                            self.dnsresource['fqdn'],
+                                            self.ip)
+
+    def create_dnsresource(self):
+        """ Create a DNS resource object
+        Due to https://bugs.launchpad.net/maas/+bug/1555393
+        This is currently unused
+        """
+        return self.maas.create_dnsresource(self.fqdn,
+                                            self.ip,
+                                            self.ttl)
+
+
+class MAASIP(object):
+    def __init__(self, options):
+        self.maas = maasclient.MAASClient(options.maas_server,
+                                          options.maas_credentials)
+        # String representation of the IP
+        self.ip = options.ip_address
+        # Dictionary representation of MAAS ipaddresss object
+        # TODO: Do this as a property
+        self.ipaddress = self.get_ipaddress()
+
+    def get_ipaddress(self):
+        """ Get an ipaddresses object """
+        ipaddresses = self.maas.get_ipaddresses()
+        self.ipaddress = None
+        for ipaddress in ipaddresses:
+            if ipaddress['ip'] == self.ip:
+                self.ipaddress = ipaddress
+        return self.ipaddress
+
+    def create_ipaddress(self, hostname=None):
+        """ Create an ipaddresses object
+        Due to https://bugs.launchpad.net/maas/+bug/1555393
+        This is currently unused
+        """
+        return self.maas.create_ipaddress(self.ip, hostname)
+
+
+def setup_logging(logfile, log_level='INFO'):
+    logFormatter = logging.Formatter(
+        fmt="%(asctime)s [%(levelname)s] %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S")
+    rootLogger = logging.getLogger()
+    rootLogger.setLevel(log_level)
+
+    consoleHandler = logging.StreamHandler()
+    consoleHandler.setFormatter(logFormatter)
+    rootLogger.addHandler(consoleHandler)
+
+    try:
+        fileLogger = logging.getLogger('file')
+        fileLogger.propagate = False
+
+        fileHandler = logging.FileHandler(logfile)
+        fileHandler.setFormatter(logFormatter)
+        rootLogger.addHandler(fileHandler)
+        fileLogger.addHandler(fileHandler)
+    except IOError:
+        logging.error('Unable to write to logfile: {}'.format(logfile))
+
+
+def dns_ha():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--maas_server', '-s',
+                        help='URL to mangage the MAAS server',
+                        required=True)
+    parser.add_argument('--maas_credentials', '-c',
+                        help='MAAS OAUTH credentials',
+                        required=True)
+    parser.add_argument('--fqdn', '-d',
+                        help='Fully Qualified Domain Name',
+                        required=True)
+    parser.add_argument('--ip_address', '-i',
+                        help='IP Address, target of the A record',
+                        required=True)
+    parser.add_argument('--ttl', '-t',
+                        help='DNS Time To Live in seconds',
+                        default='')
+    parser.add_argument('--logfile', '-l',
+                        help='Path to logfile',
+                        default='/var/log/{}.log'
+                                ''.format(sys.argv[0]
+                                          .split('/')[-1]
+                                          .split('.')[0]))
+    options = parser.parse_args()
+
+    setup_logging(options.logfile)
+    logging.info("Starting maas_dns")
+
+    dns_obj = MAASDNS(options)
+    if not dns_obj.dnsresource:
+        logging.info('DNS Resource does not exist. '
+                     'Create it with the maas cli.')
+    elif dns_obj.dnsresource.get('ip_addresses'):
+        # TODO: Handle multiple IPs returned for ip_addresses
+        for ip in dns_obj.dnsresource['ip_addresses']:
+            if ip.get('ip') != options.ip_address:
+                logging.info('Update the dnsresource with IP: {}'
+                             ''.format(options.ip_address))
+                dns_obj.update_resource()
+            else:
+                logging.info('IP is the SAME {}, no update required'
+                             ''.format(options.ip_address))
+    else:
+        logging.info('Update the dnsresource with IP: {}'
+                     ''.format(options.ip_address))
+        dns_obj.update_resource()
+
+
+if __name__ == '__main__':
+    dns_ha()

ocf/maas/maasclient/__init__.py

@@ -0,0 +1,128 @@
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from .apidriver import APIDriver
+
+log = logging.getLogger('vmaas.main')
+
+
+class MAASException(Exception):
+    pass
+
+
+class MAASDriverException(Exception):
+    pass
+
+
+class MAASClient(object):
+    """
+    A wrapper for the python maas client which makes using the API a bit
+    more user friendly.
+    """
+
+    def __init__(self, api_url, api_key, **kwargs):
+        self.driver = self._get_driver(api_url, api_key, **kwargs)
+
+    def _get_driver(self, api_url, api_key, **kwargs):
+        return APIDriver(api_url, api_key)
+
+    def _validate_maas(self):
+        try:
+            self.driver.validate_maas()
+            logging.info("Validated MAAS API")
+            return True
+        except Exception as e:
+            logging.error("MAAS API validation has failed. "
+                          "Check maas_url and maas_credentials. Error: {}"
+                          "".format(e))
+            return False
+
+    ###########################################################################
+    #  DNS API - http://maas.ubuntu.com/docs2.0/api.html#dnsresource
+    ###########################################################################
+    def get_dnsresources(self):
+        """
+        Get a listing of DNS resources which are currently defined.
+
+        :returns: a list of DNS objects
+        DNS object is a dictionary of the form:
+        {'fqdn': 'keystone.maas',
+         'resource_records': [],
+         'address_ttl': None,
+         'resource_uri': '/MAAS/api/2.0/dnsresources/1/',
+         'ip_addresses': [],
+         'id': 1}
+        """
+        resp = self.driver.get_dnsresources()
+        if resp.ok:
+            return resp.data
+        return []
+
+    def update_dnsresource(self, rid, fqdn, ip_address):
+        """
+        Updates a DNS resource with a new ip_address
+
+        :param rid: The dnsresource_id i.e.
+                    /api/2.0/dnsresources/{dnsresource_id}/
+        :param fqdn: The fqdn address to update
+        :param ip_address: The ip address to update the A record to point to
+        :returns: True if the DNS object was updated, False otherwise.
+        """
+        resp = self.driver.update_dnsresource(rid, fqdn, ip_address)
+        if resp.ok:
+            return True
+        return False
+
+    def create_dnsresource(self, fqdn, ip_address, address_ttl=None):
+        """
+        Creates a new DNS resource
+
+        :param fqdn: The fqdn address to update
+        :param ip_address: The ip address to update the A record to point to
+        :param adress_ttl: DNS time to live
+        :returns: True if the DNS object was updated, False otherwise.
+        """
+        resp = self.driver.create_dnsresource(fqdn, ip_address, address_ttl)
+        if resp.ok:
+            return True
+        return False
+
+    ###########################################################################
+    #  IP API - http://maas.ubuntu.com/docs2.0/api.html#ip-address
+    ###########################################################################
+    def get_ipaddresses(self):
+        """
+        Get a list of ip addresses
+
+        :returns: a list of ip address dictionaries
+        """
+        resp = self.driver.get_ipaddresses()
+        if resp.ok:
+            return resp.data
+        return []
+
+    def create_ipaddress(self, ip_address, hostname=None):
+        """
+        Creates a new IP resource
+
+        :param ip_address: The ip address to register
+        :param hostname: the hostname to register at the same time
+        :returns: True if the DNS object was updated, False otherwise.
+        """
+        resp = self.driver.create_ipaddress(ip_address, hostname)
+        if resp.ok:
+            return True
+        return False

ocf/maas/maasclient/apidriver.py

@@ -0,0 +1,211 @@
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import yaml
+import logging
+
+from apiclient import maas_client as maas
+from .driver import MAASDriver
+from .driver import Response
+
+try:
+    from urllib2 import HTTPError
+except ImportError:
+    from urllib3.exceptions import HTTPError
+
+log = logging.getLogger('vmaas.main')
+OK = 200
+
+
+class APIDriver(MAASDriver):
+    """
+    A MAAS driver implementation which uses the MAAS API.
+    """
+
+    def __init__(self, api_url, api_key, *args, **kwargs):
+        if api_url[-1] != '/':
+            api_url += '/'
+        if api_url.find('/api/') < 0:
+            api_url = api_url + 'api/2.0/'
+        super(APIDriver, self).__init__(api_url, api_key, *args, **kwargs)
+        self._client = None
+        self._oauth = None
+
+    @property
+    def client(self):
+        """
+        MAAS client
+
+        :rtype: MAASClient
+        """
+        if self._client:
+            return self._client
+
+        self._client = maas.MAASClient(auth=self.oauth,
+                                       dispatcher=maas.MAASDispatcher(),
+                                       base_url=self.api_url)
+        return self._client
+
+    @property
+    def oauth(self):
+        """
+        MAAS OAuth information for interacting with the MAAS API.
+
+        :rtype: MAASOAuth
+        """
+        if self._oauth:
+            return self._oauth
+
+        if self.api_key:
+            api_key = self.api_key.split(':')
+            self._oauth = maas.MAASOAuth(consumer_key=api_key[0],
+                                         resource_token=api_key[1],
+                                         resource_secret=api_key[2])
+            return self._oauth
+        else:
+            return None
+
+    def validate_maas(self):
+        return self._get('/')
+
+    def _get(self, path, **kwargs):
+        """
+        Issues a GET request to the MAAS REST API, returning the data
+        from the query in the python form of the json data.
+        """
+        response = self.client.get(path, **kwargs)
+        payload = response.read()
+        log.debug("Request %s results: [%s] %s", path, response.getcode(),
+                  payload)
+
+        if response.getcode() == OK:
+            return Response(True, yaml.load(payload))
+        else:
+            return Response(False, payload)
+
+    def _post(self, path, op='update', **kwargs):
+        """
+        Issues a POST request to the MAAS REST API.
+        """
+        try:
+            response = self.client.post(path, **kwargs)
+            payload = response.read()
+            log.debug("Request %s results: [%s] %s", path, response.getcode(),
+                      payload)
+
+            if response.getcode() == OK:
+                return Response(True, yaml.load(payload))
+            else:
+                return Response(False, payload)
+        except HTTPError as e:
+            log.error("Error encountered: %s for %s with params %s",
+                      str(e), path, str(kwargs))
+            return Response(False, None)
+        except Exception as e:
+            log.error("Post request raised exception: %s", e)
+            return Response(False, None)
+
+    def _put(self, path, **kwargs):
+        """
+        Issues a PUT request to the MAAS REST API.
+        """
+        try:
+            response = self.client.put(path, **kwargs)
+            payload = response.read()
+            log.debug("Request %s results: [%s] %s", path, response.getcode(),
+                      payload)
+            if response.getcode() == OK:
+                return Response(True, payload)
+            else:
+                return Response(False, payload)
+        except HTTPError as e:
+            log.error("Error encountered: %s with details: %s for %s with "
+                      "params %s", e, e.read(), path, str(kwargs))
+            return Response(False, None)
+        except Exception as e:
+            log.error("Put request raised exception: %s", e)
+            return Response(False, None)
+
+    ###########################################################################
+    #  DNS API - http://maas.ubuntu.com/docs2.0/api.html#dnsresource
+    ###########################################################################
+    def get_dnsresources(self):
+        """
+        Get a listing of the MAAS dnsresources
+
+        :returns: a list of MAAS dnsresrouce objects
+        """
+        return self._get('/dnsresources/')
+
+    def update_dnsresource(self, rid, fqdn, ip_address):
+        """
+        Updates a DNS resource with a new ip_address
+
+        :param rid: The dnsresource_id i.e.
+                    /api/2.0/dnsresources/{dnsresource_id}/
+        :param fqdn: The fqdn address to update
+        :param ip_address: The ip address to update the A record to point to
+        :returns: True if the DNS object was updated, False otherwise.
+        """
+        return self._put('/dnsresources/{}/'.format(rid), fqdn=fqdn,
+                         ip_addresses=ip_address)
+
+    def create_dnsresource(self, fqdn, ip_address, address_ttl=None):
+        """
+        Creates a new DNS resource
+
+        :param fqdn: The fqdn address to update
+        :param ip_address: The ip address to update the A record to point to
+        :param adress_ttl: DNS time to live
+        :returns: True if the DNS object was updated, False otherwise.
+        """
+        fqdn = bytes(fqdn, encoding='utf-8')
+        ip_address = bytes(ip_address, encoding='utf-8')
+        if address_ttl:
+            return self._post('/dnsresources/',
+                              fqdn=fqdn,
+                              ip_addresses=ip_address,
+                              address_ttl=address_ttl)
+        else:
+            return self._post('/dnsresources/',
+                              fqdn=fqdn,
+                              ip_addresses=ip_address)
+
+    ###########################################################################
+    #  IP API - http://maas.ubuntu.com/docs2.0/api.html#ip-addresses
+    ###########################################################################
+    def get_ipaddresses(self):
+        """
+        Get a dictionary of a given ip_address
+
+        :param ip_address: The ip address to get information for
+        :returns: a dictionary for a given ip
+        """
+        return self._get('/ipaddresses/')
+
+    def create_ipaddress(self, ip_address, hostname=None):
+        """
+        Creates a new IP resource
+
+        :param ip_address: The ip address to register
+        :param hostname: the hostname to register at the same time
+        :returns: True if the DNS object was updated, False otherwise.
+        """
+        if hostname:
+            return self._post('/ipaddresses/', op='reserve',
+                              ip_addresses=ip_address,
+                              hostname=hostname)
+        else:
+            return self._post('/ipaddresses/', op='reserve',
+                              ip_addresses=ip_address)

ocf/maas/maasclient/driver.py

@@ -0,0 +1,63 @@
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+log = logging.getLogger('vmaas.main')
+
+
+class Response(object):
+    """
+    Response for the API calls to use internally
+    """
+    def __init__(self, ok=False, data=None):
+        self.ok = ok
+        self.data = data
+
+    def __nonzero__(self):
+        """Allow boolean comparison"""
+        return bool(self.ok)
+
+
+class MAASDriver(object):
+    """
+    Defines the commands and interfaces for generically working with
+    the MAAS controllers.
+    """
+
+    def __init__(self, api_url, api_key):
+        self.api_url = api_url
+        self.api_key = api_key
+
+    def _get_system_id(self, obj):
+        """
+        Returns the system_id from an object or the object itself
+        if the system_id is not found.
+        """
+        if 'system_id' in obj:
+            return obj.system_id
+        return obj
+
+    def _get_uuid(self, obj):
+        """
+        Returns the UUID for the MAAS object. If the object has the attribute
+        'uuid', then this method will return obj.uuid, otherwise this method
+        will return the object itself.
+        """
+        if hasattr(obj, 'uuid'):
+            return obj.uuid
+        else:
+            log.warning("Attr 'uuid' not found in %s" % obj)
+
+        return obj

unit_tests/test_hacluster_hooks.py

@@ -93,3 +93,130 @@ class TestCorosyncConf(unittest.TestCase):
                 else:
                     commit.assert_any_call(
                         'crm -w -F configure %s %s %s' % (kw, name, params))
+
+    @mock.patch.object(hooks, 'setup_maas_api')
+    @mock.patch.object(hooks, 'validate_dns_ha')
+    @mock.patch('pcmk.wait_for_pcmk')
+    @mock.patch.object(hooks, 'peer_units')
+    @mock.patch('pcmk.crm_opt_exists')
+    @mock.patch.object(hooks, 'oldest_peer')
+    @mock.patch.object(hooks, 'configure_corosync')
+    @mock.patch.object(hooks, 'configure_cluster_global')
+    @mock.patch.object(hooks, 'configure_monitor_host')
+    @mock.patch.object(hooks, 'configure_stonith')
+    @mock.patch.object(hooks, 'related_units')
+    @mock.patch.object(hooks, 'get_cluster_nodes')
+    @mock.patch.object(hooks, 'relation_set')
+    @mock.patch.object(hooks, 'relation_ids')
+    @mock.patch.object(hooks, 'get_corosync_conf')
+    @mock.patch('pcmk.commit')
+    @mock.patch.object(hooks, 'config')
+    @mock.patch.object(hooks, 'parse_data')
+    def test_ha_relation_changed_dns_ha(self, parse_data, config, commit,
+                                        get_corosync_conf, relation_ids,
+                                        relation_set, get_cluster_nodes,
+                                        related_units, configure_stonith,
+                                        configure_monitor_host,
+                                        configure_cluster_global,
+                                        configure_corosync, oldest_peer,
+                                        crm_opt_exists, peer_units,
+                                        wait_for_pcmk, validate_dns_ha,
+                                        setup_maas_api):
+        validate_dns_ha.return_value = True
+        crm_opt_exists.return_value = False
+        oldest_peer.return_value = True
+        related_units.return_value = ['ha/0', 'ha/1', 'ha/2']
+        get_cluster_nodes.return_value = ['10.0.3.2', '10.0.3.3', '10.0.3.4']
+        relation_ids.return_value = ['ha:1']
+        get_corosync_conf.return_value = True
+        cfg = {'debug': False,
+               'prefer-ipv6': False,
+               'corosync_transport': 'udpu',
+               'corosync_mcastaddr': 'corosync_mcastaddr',
+               'cluster_count': 3,
+               'maas_url': 'http://maas/MAAAS/',
+               'maas_credentials': 'secret'}
+
+        config.side_effect = lambda key: cfg.get(key)
+
+        rel_get_data = {'locations': {'loc_foo': 'bar rule inf: meh eq 1'},
+                        'clones': {'cl_foo': 'res_foo meta interleave=true'},
+                        'groups': {'grp_foo': 'res_foo'},
+                        'colocations': {'co_foo': 'inf: grp_foo cl_foo'},
+                        'resources': {'res_foo_hostname': 'ocf:maas:dns'},
+                        'resource_params': {'res_foo_hostname': 'params bar'},
+                        'ms': {'ms_foo': 'res_foo meta notify=true'},
+                        'orders': {'foo_after': 'inf: res_foo ms_foo'}}
+
+        def fake_parse_data(relid, unit, key):
+            return rel_get_data.get(key, {})
+
+        parse_data.side_effect = fake_parse_data
+
+        hooks.ha_relation_changed()
+        self.assertTrue(validate_dns_ha.called)
+        self.assertTrue(setup_maas_api.called)
+        # Validate maas_credentials and maas_url are added to params
+        commit.assert_any_call(
+            'crm -w -F configure primitive res_foo_hostname ocf:maas:dns '
+            'params bar maas_url="http://maas/MAAAS/" '
+            'maas_credentials="secret"')
+
+    @mock.patch.object(hooks, 'setup_maas_api')
+    @mock.patch.object(hooks, 'validate_dns_ha')
+    @mock.patch('pcmk.wait_for_pcmk')
+    @mock.patch.object(hooks, 'peer_units')
+    @mock.patch('pcmk.crm_opt_exists')
+    @mock.patch.object(hooks, 'oldest_peer')
+    @mock.patch.object(hooks, 'configure_corosync')
+    @mock.patch.object(hooks, 'configure_cluster_global')
+    @mock.patch.object(hooks, 'configure_monitor_host')
+    @mock.patch.object(hooks, 'configure_stonith')
+    @mock.patch.object(hooks, 'related_units')
+    @mock.patch.object(hooks, 'get_cluster_nodes')
+    @mock.patch.object(hooks, 'relation_set')
+    @mock.patch.object(hooks, 'relation_ids')
+    @mock.patch.object(hooks, 'get_corosync_conf')
+    @mock.patch('pcmk.commit')
+    @mock.patch.object(hooks, 'config')
+    @mock.patch.object(hooks, 'parse_data')
+    def test_ha_relation_changed_dns_ha_missing(
+            self, parse_data, config, commit, get_corosync_conf, relation_ids,
+            relation_set, get_cluster_nodes, related_units, configure_stonith,
+            configure_monitor_host, configure_cluster_global,
+            configure_corosync, oldest_peer, crm_opt_exists, peer_units,
+            wait_for_pcmk, validate_dns_ha, setup_maas_api):
+
+        validate_dns_ha.return_value = False
+        crm_opt_exists.return_value = False
+        oldest_peer.return_value = True
+        related_units.return_value = ['ha/0', 'ha/1', 'ha/2']
+        get_cluster_nodes.return_value = ['10.0.3.2', '10.0.3.3', '10.0.3.4']
+        relation_ids.return_value = ['ha:1']
+        get_corosync_conf.return_value = True
+        cfg = {'debug': False,
+               'prefer-ipv6': False,
+               'corosync_transport': 'udpu',
+               'corosync_mcastaddr': 'corosync_mcastaddr',
+               'cluster_count': 3,
+               'maas_url': 'http://maas/MAAAS/',
+               'maas_credentials': None}
+
+        config.side_effect = lambda key: cfg.get(key)
+
+        rel_get_data = {'locations': {'loc_foo': 'bar rule inf: meh eq 1'},
+                        'clones': {'cl_foo': 'res_foo meta interleave=true'},
+                        'groups': {'grp_foo': 'res_foo'},
+                        'colocations': {'co_foo': 'inf: grp_foo cl_foo'},
+                        'resources': {'res_foo_hostname': 'ocf:maas:dns'},
+                        'resource_params': {'res_foo_hostname': 'params bar'},
+                        'ms': {'ms_foo': 'res_foo meta notify=true'},
+                        'orders': {'foo_after': 'inf: res_foo ms_foo'}}
+
+        def fake_parse_data(relid, unit, key):
+            return rel_get_data.get(key, {})
+
+        parse_data.side_effect = fake_parse_data
+
+        with self.assertRaises(ValueError):
+            hooks.ha_relation_changed()

unit_tests/test_hacluster_utils.py

@@ -146,3 +146,39 @@ class UtilsTestCase(unittest.TestCase):
 
         self.assertFalse(mock_get_host_ip.called)
         self.assertTrue(mock_get_ipv6_addr.called)
+
+    @mock.patch.object(utils, 'assert_charm_supports_dns_ha')
+    @mock.patch.object(utils, 'config')
+    def test_validate_dns_ha_valid(self, config,
+                                   assert_charm_supports_dns_ha):
+        cfg = {'maas_url': 'http://maas/MAAAS/',
+               'maas_credentials': 'secret'}
+        config.side_effect = lambda key: cfg.get(key)
+
+        self.assertTrue(utils.validate_dns_ha())
+        self.assertTrue(assert_charm_supports_dns_ha.called)
+
+    @mock.patch.object(utils, 'assert_charm_supports_dns_ha')
+    @mock.patch.object(utils, 'status_set')
+    @mock.patch.object(utils, 'config')
+    def test_validate_dns_ha_invalid(self, config, status_set,
+                                     assert_charm_supports_dns_ha):
+        cfg = {'maas_url': 'http://maas/MAAAS/',
+               'maas_credentials': None}
+        config.side_effect = lambda key: cfg.get(key)
+
+        self.assertRaises(utils.MAASConfigIncomplete,
+                          lambda: utils.validate_dns_ha())
+        self.assertTrue(assert_charm_supports_dns_ha.called)
+
+    @mock.patch.object(utils, 'apt_install')
+    @mock.patch.object(utils, 'apt_update')
+    @mock.patch.object(utils, 'add_source')
+    @mock.patch.object(utils, 'config')
+    def test_setup_maas_api(self, config, add_source, apt_update, apt_install):
+        cfg = {'maas_source': 'ppa:maas/stable'}
+        config.side_effect = lambda key: cfg.get(key)
+
+        utils.setup_maas_api()
+        add_source.assert_called_with(cfg['maas_source'])
+        self.assertTrue(apt_install.called)